TheMajesticNetwork/Bolt-Pet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

feat: Enhance local command handling and introduce local intents

Browse Source 
- Refactor `run_local_command` to manage subprocesses more effectively, ensuring child processes are terminated on timeout.
- Introduce `_terminate` function to handle process group termination and capture output.
- Implement `_command_output` to format command results with a character limit.
- Add local intent recognition in `intents.py` to handle commands like "stop", "go to sleep", and "come here" without server interaction.
- Normalize user input to match local intents while stripping filler words.
- Update tests to cover new local intent functionality and ensure proper command handling.
- Enhance speech processing to handle abbreviations and improve spoken output clarity.
This commit is contained in:
Aaron Franks
2026-08-05 18:31:02 -06:00
parent 8d4751d80f
commit 3ee67cb4d6
14 changed files with 1107 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bolt_pet/controller.py
+193 -27
View File
@@ -15,15 +15,16 @@ from __future__ import annotations
import threading
import time
from collections import deque
from typing import Optional
from PySide6.QtCore import QObject, Signal
from . import (
config, dialogue as dialogue_mod, file_delivery, file_ops,
history as history_mod, monitors as monitors_mod, notifications,
pet_actions, quiet, screen_context, screen_text, self_restart,
server_client, speech_text, updater,
history as history_mod, intents as intents_mod, monitors as monitors_mod,
notifications, pet_actions, quiet, screen_context, screen_text,
self_restart, server_client, speech_text, updater,
)
from . import __version__
from .audio import barge_in, mic, stt, tts, wake_word
@@ -98,7 +99,14 @@ class PetController(QObject):
self._notification_gate = notifications.NotificationGate(
config.NOTIFICATION_FILTER, config.NOTIFICATION_MIN_INTERVAL_SECONDS
)
self._pending_notifications: list[notifications.Notification] = []
# Bounded, and stamped on arrival: the drain only runs from the
# heartbeat, which doesn't run while napping, so this fills up
# overnight. maxlen drops the oldest rather than growing without limit,
# and the stamp lets the drain discard a backlog nobody wants read out
# at 8am (see _drain_notifications).
self._pending_notifications: deque[tuple[float, notifications.Notification]] = deque(
maxlen=max(1, config.NOTIFICATION_QUEUE_LIMIT)
)
self._notification_lock = threading.Lock()
# ── external controls (safe to call from the Qt/UI thread) ─────────
@@ -187,18 +195,47 @@ class PetController(QObject):
)
self.log.emit(f"Barge-in: {config.BARGE_IN_MODE} mode.")
with self._stream:
try:
health = server_client.check_health()
self.log.emit(f"Connected to server: {health}")
except Exception as exc:
self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.")
self._start_notification_bridge()
self._report_self_restart()
self._loop()
if self._notification_watcher is not None:
self._notification_watcher.stop()
self.finished.emit()
# Everything past here is in try/finally because `finished` is what
# ui/app.py waits on to quit the QThread and to run a pending
# os.execv. An exception escaping _loop used to skip it, leaving the
# thread wedged with the mic still open and no restart — so the failure
# mode of any bug below was "the pet goes deaf and the tray won't quit"
# rather than "one turn failed".
try:
with self._stream:
try:
health = server_client.check_health()
self.log.emit(f"Connected to server: {health}")
except Exception as exc:
self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.")
self._start_notification_bridge()
self._guarded(self._report_self_restart, "restart report")
self._loop()
except Exception as exc:
self.log.emit(f"Pipeline stopped unexpectedly: {exc!r}")
finally:
if self._notification_watcher is not None:
self._notification_watcher.stop()
self.finished.emit()
def _guarded(self, work, label: str) -> bool:
"""Run *work*, absorbing anything it raises.
The pipeline is one thread driving a state machine that raises on an
illegal transition (deliberately — see state.py), plus a dozen
best-effort subsystems that shell out, hit the network, or touch the
filesystem. Any one of them raising something unforeseen used to end the
whole session. Here, it costs a log line and a forced return to IDLE,
which is the only state it's always safe to resume from.
Returns True if *work* completed without raising."""
try:
work()
return True
except Exception as exc:
self.log.emit(f"Recovered from a {label} failure: {exc!r}")
self._state.force(PetState.IDLE)
return False
def _loop(self) -> None:
while self._running:
@@ -214,7 +251,7 @@ class PetController(QObject):
if not self._running:
return
continue
self._handle_conversation_turn()
self._guarded(self._handle_conversation_turn, "conversation turn")
def _wait_for_wake_or_click(self) -> bool:
"""True once either the wake phrase was heard or a click-to-talk
@@ -226,7 +263,12 @@ class PetController(QObject):
self._stream,
should_continue=should_continue,
threshold=self.wake_threshold, # callable: the tuner slider is live
on_tick=self._maybe_heartbeat,
# Guarded: on_tick is the one place control returns to us during a
# listen that can block for minutes, and everything it drives
# (update check, nap probe, notification forwarding) touches the
# network or shells out. Unguarded, any of them raising would unwind
# the listen loop and end the session.
on_tick=lambda: self._guarded(self._maybe_heartbeat, "heartbeat"),
on_score=self._observe_wake_score,
)
if not self._running:
@@ -275,6 +317,13 @@ class PetController(QObject):
self.log.emit(f"You: {text}")
self.history.add(history_mod.USER, text, time.time())
# "stop", "come here", "say that again" — answered here, without the
# round trip. Never on a follow-up turn: Bolt asked you something and
# the answer is his, even if it happens to look like a body command.
if not following_up and self._handle_local_intent(text):
self._state.transition(PetState.IDLE)
return
try:
# What's focused right now rides along, so "what's this error?"
# has a referent without you having to describe the window.
@@ -293,6 +342,60 @@ class PetController(QObject):
self._state.transition(PetState.IDLE)
self._maybe_self_restart()
def _handle_local_intent(self, text: str) -> bool:
"""Answer *text* locally if it's one of the closed set of body commands
in intents.py. Returns True if it was handled (no server call).
The effects live here rather than in intents.py for the same reason
pet_actions splits parse from describe: recognising the phrase is pure
and testable, doing the thing needs the controller's state, the tray's
nap override and a Qt signal to the window."""
if not config.LOCAL_INTENTS:
return False
intent = intents_mod.recognize(text)
if intent is None:
return False
self.log.emit(f"Local intent: {intent.name} (answered without the server)")
if intent.name == "stop":
# Nothing to say and nothing to do: silence is the acknowledgement.
# Also ends any follow-up chain — "never mind" means the
# conversation is over, not that we should keep the mic open.
self._follow_ups = 0
self._pending_follow_up = False
self._talk_now.clear()
return True
if intent.name == "repeat":
last = self.history.last(history_mod.PET)
if last is None:
self._speak("I haven't said anything yet.")
else:
# remember=False: replaying a line isn't a new turn. Appending it
# would make "say that again" twice over read back as a
# conversation where Bolt volunteered the same thing three times.
self._speak(last.text, remember=False)
return True
if intent.name == "voice_reset":
had_voice = bool(self._voice_id)
self.reset_voice()
self._speak(intent.speak if had_voice else "That is my normal voice.")
return True
action = intent.action
if action is not None:
if action.get("action") == "nap":
# Through set_napping, not just the signal, so a spoken "go to
# sleep" overrides the quiet-hours schedule exactly like the
# tray's Nap entry and `petctl nap` do — otherwise the next
# schedule check would undo it within ten seconds.
self.set_napping(bool(action["enabled"]))
self.action.emit(dict(action))
if intent.speak:
self._speak(intent.speak)
return True
def _with_context(self, text: str) -> str:
"""Everything the server gets alongside what you actually said: the
focused window title, and a one-line note about the screen layout so
@@ -319,9 +422,26 @@ class PetController(QObject):
self._pet_monitor = int(index)
def _handle_command(self, command: str) -> str:
"""Server-relayed command. `petctl ...` drives the pet's body and
`filectl ...` does local file read/write/edit — neither ever reaches
a shell; everything else is a real command, exactly as before (see
"""Server-relayed command, with the guarantee the relay depends on: this
always returns a string.
The server is blocked on `/desk/tool_result` while this runs. If it
raises instead of answering, the relay never posts, the turn dies
mid-flight, and the server sits out its own timeout on a conversation it
can't finish — the worst available failure mode, because it's silent on
both ends. Handing the exception back as command output instead means
Bolt can read what went wrong and say so, or try something else, inside
the same turn."""
try:
return self._dispatch_command(command)
except Exception as exc:
self.log.emit(f"Command handler failed: {exc!r}")
return f"[error] the pet couldn't run that: {exc}"
def _dispatch_command(self, command: str) -> str:
"""`petctl ...` drives the pet's body, `dialoguectl ...` plays a scene
and `filectl ...` does local file read/write/edit — none of them ever
reach a shell; everything else is a real command, exactly as before (see
the security notes in the README)."""
try:
action = pet_actions.parse(command)
@@ -568,14 +688,15 @@ class PetController(QObject):
elif not config.VOICE_STICKY:
self.reset_voice()
def _speak(self, text: str) -> None:
def _speak(self, text: str, remember: bool = True) -> None:
self._state.transition(PetState.TALKING)
# Bubble gets the markdown stripped but emoji kept (it can't render
# **bold** but draws emoji fine); tts.speak() does its own, stricter
# sanitizing for the voice.
self.said.emit(speech_text.for_display(text))
self.log.emit(f"Bolt: {text}")
self.history.add(history_mod.PET, text, time.time())
if remember:
self.history.add(history_mod.PET, text, time.time())
should_stop = None
if self._barge_in is not None:
@@ -608,6 +729,15 @@ class PetController(QObject):
f"Asked a question — listening for your answer "
f"({self._follow_ups}{'/' + str(cap) if cap > 0 else ''})."
)
# The tail of the reply we just played is still in the mic's ring
# buffer, and we're about to start recording with a VAD that will
# take it for the start of your answer — Bolt's own last words,
# transcribed and sent back to him as if you'd said them. Nothing you
# said can be in there: playback ran to completion, so if you had
# spoken, barge-in would have cut it and taken the other branch.
dropped = mic.flush(self._stream)
if dropped:
self.log.emit(f"Dropped {dropped} buffered frames of my own voice.")
self._pending_follow_up = True
self._talk_now.set()
@@ -686,16 +816,39 @@ class PetController(QObject):
def _queue_notification(self, notification: notifications.Notification) -> None:
"""Called on the watcher thread — just queue it; forwarding happens on
the pipeline thread where it can't collide with a live conversation."""
if not self._notification_gate.should_forward(notification, time.monotonic()):
now = time.monotonic()
if not self._notification_gate.should_forward(notification, now):
return
with self._notification_lock:
self._pending_notifications.append(notification)
if len(self._pending_notifications) == self._pending_notifications.maxlen:
# Say so rather than dropping in silence: a full queue means the
# bridge is matching more than the pet can plausibly speak, and
# the filter is what wants tightening.
self.log.emit("Notification queue full — dropping the oldest.")
self._pending_notifications.append((now, notification))
def _drain_notifications(self) -> None:
with self._notification_lock:
pending, self._pending_notifications = self._pending_notifications, []
for notification in pending:
pending = list(self._pending_notifications)
self._pending_notifications.clear()
now = time.monotonic()
max_age = config.NOTIFICATION_MAX_AGE_SECONDS
if max_age > 0:
fresh = [entry for entry in pending if now - entry[0] <= max_age]
if len(fresh) != len(pending):
self.log.emit(
f"Skipping {len(pending) - len(fresh)} notification(s) older than "
f"{int(max_age)}s."
)
pending = fresh
for index, (_stamped, notification) in enumerate(pending):
if not self._running or self._napping:
# Put back what we haven't forwarded — the old code swapped the
# queue out and then returned, silently dropping the remainder
# the moment a nap started mid-drain.
self._requeue_notifications(pending[index:])
return
self.log.emit(f"Notification: {notification.as_text()}")
self.history.add(history_mod.SYSTEM, notification.as_text(), time.time())
@@ -705,7 +858,11 @@ class PetController(QObject):
on_command=self._handle_command,
)
except server_client.ServerError as exc:
# Keep this one and everything behind it for the next heartbeat:
# the server being briefly down shouldn't silently eat the
# backlog. The age limit is what stops that retrying forever.
self.log.emit(f"Couldn't forward notification: {exc}")
self._requeue_notifications(pending[index:])
return
self._check_deliveries()
self._apply_voice(reply)
@@ -713,6 +870,15 @@ class PetController(QObject):
self._speak(reply.text)
self._state.transition(PetState.IDLE)
def _requeue_notifications(self, entries: list) -> None:
"""Push undelivered notifications back on the front, oldest first, so a
retry keeps their original order (and their original timestamps, so a
retry loop can't keep a stale one alive indefinitely)."""
if not entries:
return
with self._notification_lock:
self._pending_notifications.extendleft(reversed(entries))
# ── file delivery ────────────────────────────────────────────────────
def _check_deliveries(self) -> None: