diff --git a/.claude/settings.local.json b/.claude/settings.local.json index ed0f8c1..44fae21 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -89,7 +89,10 @@ "Bash(dig +short themajesticnetwork.com)", "Bash(dig +short api.themajesticnetwork.com)", "Bash(timeout 900 /tmp/claude-1000/-home-maji-Documents-Bolt-Pet/58f4a7c3-6a92-47ed-9e43-bb2216d8b135/scratchpad/tmnvenv/bin/python -m pytest tests/test_site.py -q -p no:cacheprovider)", - "Bash(curl -s -o /dev/null -w 'HTTP %{http_code} bytes=%{size_download}\\\\n' -m 15 -H 'X-Forwarded-For: 1.2.3.4' -H 'X-Real-IP: 1.2.3.4' -A 'Mozilla/5.0 \\(X11; Linux x86_64\\) Firefox/152.0' https://themajesticnetwork.com/?claude-probe-__TRACKED_VAR__)" + "Bash(curl -s -o /dev/null -w 'HTTP %{http_code} bytes=%{size_download}\\\\n' -m 15 -H 'X-Forwarded-For: 1.2.3.4' -H 'X-Real-IP: 1.2.3.4' -A 'Mozilla/5.0 \\(X11; Linux x86_64\\) Firefox/152.0' https://themajesticnetwork.com/?claude-probe-__TRACKED_VAR__)", + "Bash(QT_QPA_PLATFORM=offscreen timeout 300 .venv/bin/pytest tests/ -q)", + "Bash(QT_QPA_PLATFORM=offscreen timeout 120 .venv/bin/pytest tests/test_intents.py -q)", + "Bash(QT_QPA_PLATFORM=offscreen timeout 120 .venv/bin/pytest tests/test_speech_text.py -q)" ] } } diff --git a/.env.example b/.env.example index c0da76b..dac03dc 100644 --- a/.env.example +++ b/.env.example @@ -72,8 +72,17 @@ ELEVENLABS_VOICE_ID= #VAD_MIN_UTTERANCE_SECONDS=0.4 #VAD_GRACE_SECONDS=4 # how long to wait for you to start talking +# ── Local intents (optional) ──────────────────────────────────────────────── +# A short, closed list of things the pet answers itself, with no server round +# trip: "stop", "be quiet", "come here", "go away", "go to sleep", "wake up", +# "say that again", "sit"/"stay", "go for a walk", "use your normal voice". +# Matched whole and exact, and never while you're answering a question Bolt +# asked, so a real request ("stop the docker container") still goes to him. +# Set to false to route absolutely everything through the server. +#LOCAL_INTENTS=true + # ── Follow-up listening (optional) ────────────────────────────────────────── -# When a reply ends on a question, the pet keeps listening for your answer +# When a reply asks you something, the pet keeps listening for your answer # instead of dropping back to idle and making you say the wake word again. # FOLLOW_UP_MAX_TURNS caps how many question-and-answer rounds can chain # without you re-triggering it (0 = no cap) — a stop on runaway loops if the @@ -181,6 +190,12 @@ ELEVENLABS_VOICE_ID= #NOTIFICATION_BRIDGE=false #NOTIFICATION_FILTER=build|deploy|calendar #NOTIFICATION_MIN_INTERVAL_SECONDS=60 +# Notifications queue while the pet is napping (the heartbeat that forwards them +# doesn't run). These two stop an overnight backlog becoming a monologue at 8am: +# the queue drops its oldest past the limit, and anything staler than the age +# limit is discarded rather than read out. +#NOTIFICATION_QUEUE_LIMIT=20 +#NOTIFICATION_MAX_AGE_SECONDS=900 # ── Push-to-talk (optional) ───────────────────────────────────────────────── # Global hotkey; needs pynput and a session that allows global key hooks diff --git a/CLAUDE.md b/CLAUDE.md index 727f890..379042a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,12 +79,41 @@ logs a missing-config message and exits its thread instead of starting. queued desktop notifications. It owns the live wake-word threshold (`wake_threshold()` is passed to `listen_for_wake_word` as a *callable* so the tray slider takes effect mid-listen) and the conversation `history`. + + **One thread drives all of it, so failure containment is structural.** Every + entry point that can raise runs inside `_guarded(work, label)`, which logs and + forces the machine back to IDLE (the only state it's always safe to resume + from): the conversation turn, the heartbeat tick — which matters most, since + `on_tick` is the one place control returns to us during a listen that blocks + for minutes, and everything it drives touches the network or shells out — and + the post-restart report. `run()` wraps the lot in try/finally because + `finished` is what `ui/app.py` waits on to quit the thread and to run a + pending `os.execv`; an exception escaping `_loop` used to skip it, so the + failure mode of any bug below was "the pet goes deaf with the mic still open + and the tray won't quit" rather than "one turn failed". `_handle_command` has + the same shape for a different reason: it must **always return a string**, + because the server is blocked on `/desk/tool_result` while it runs and an + exception there means the relay never posts and the server sits out its own + timeout on a turn that can't finish — silent on both ends. Handed back as + command output instead, Bolt can read what broke and say so in the same turn. - **`server_client.py`** — HTTP client for the desk API, dependency-free beyond `requests` so it's easy to mock in tests. `converse()` loops relaying - server-issued shell commands (`run_local_command`, executed via - `subprocess.run(shell=True)` as the desktop user, 30s default timeout) via + server-issued shell commands (`run_local_command`, executed via a + `shell=True` `Popen` as the desktop user, 30s default timeout) via `/desk/tool_result` until the server sends a final `reply` (capped at - `_MAX_RELAY_HOPS`). This is the same "full desktop control" trust model as + `_MAX_RELAY_HOPS` — exhausting which is reported as its own error, because + "unknown server response" sent everyone looking at the payload shape when what + happened is a model that kept calling tools and never answered). + `run_local_command` is `Popen` rather than `subprocess.run` for the timeout + path: the command is a shell, and `run()`'s timeout would kill only that + shell, leaving whatever it spawned (a build, a `tail -f`, an ffmpeg) alive for + the rest of the session with no parent watching — so the child gets its own + process group (`start_new_session`, POSIX) and a timeout SIGTERMs the group, + SIGKILLs it two seconds later, then drains the pipes *with its own timeout* so + a grandchild holding stdout can't turn a timeout into a hang. Whatever the + command printed before it hung is returned alongside the timeout notice, since + the last line usually says exactly what it was stuck waiting for. This is the + same "full desktop control" trust model as the server repo's other desk clients — commands only ever originate from the user's own voice/click requests in their own session. A final reply is returned as a `Reply(text, voice_id, voice_name)` rather than a bare string, @@ -108,7 +137,8 @@ logs a missing-config message and exits its thread instead of starting. the name isn't always coming from someone as trusted as the owner. Toggle off entirely with `RECEIVE_FILES=false`. - **`audio/`** — `mic.py` (energy-based VAD utterance capture, ported from the - server repo's `bolt_desk.py`), `wake_word.py` (openWakeWord `thunderbolt.onnx` + server repo's `bolt_desk.py`, plus `flush()` — see the note below on the pet + hearing itself), `wake_word.py` (openWakeWord `thunderbolt.onnx` detection + `NearMissLog` for threshold tuning — see below), `stt.py` (Deepgram), `tts.py` (ElevenLabs, streaming by default — `stream_pcm()` + `play_stream()` start playback on the first chunk; `chunks_to_int16()` @@ -266,6 +296,14 @@ logs a missing-config message and exits its thread instead of starting. `dbus-monitor`, parses Notify calls (pure `iter_notifications()`), filters and rate-limits them (`NotificationGate`), and the controller forwards survivors through `converse()`. Off by default — each one is a round trip. + Note where the queue between the two threads lives: notifications arrive on + the watcher thread and are forwarded from the heartbeat, which **doesn't run + while the pet is napping** — so they accumulate overnight. The controller's + queue is therefore a bounded `deque` stamped on arrival, and the drain + discards anything older than `NOTIFICATION_MAX_AGE_SECONDS` rather than + reading a nine-hour-old backlog out at 8am. A drain that stops early (a nap + starting mid-loop, or the server going down) re-queues what it didn't forward + instead of dropping it, which the original swap-and-return did silently. - **`sudo_askpass.py`** — makes server-relayed `sudo` usable from a process with no terminal, by pointing sudo's `SUDO_ASKPASS` at a GUI helper and rewriting bare `sudo` to `sudo -A` (`add_askpass_flag`, a conservative regex @@ -302,15 +340,43 @@ logs a missing-config message and exits its thread instead of starting. `for_speech()` (called inside `tts.speak()`, so every path to the speakers is covered) strips markdown, emoji, URLs and stray symbols the voice would read literally ("asterisk asterisk"), turns bullet lists into full sentences, and - words a few symbols (`&` → "and"). `for_display()` is the looser version for + words a few symbols (`&` → "and"), abbreviations the voice would spell out + letter by letter (`e.g.` → "for example", `etc.` → "and so on") and a long + option's leading `--` (heard as "dash dash force"; the single hyphen has to + survive for "bolt-pet"). `for_display()` is the looser version for the speech bubble — markdown syntax gone, emoji kept. `is_question()` decides whether a reply leaves the pet waiting on an answer: it tests the *spoken* - form (so a '?' inside a stripped code block or URL doesn't count) and only a - trailing one counts, since a question asked in passing isn't awaiting a - reply. `controller._should_follow_up` uses it to keep listening without the + form (so a '?' inside a stripped code block or URL doesn't count) and a '?' + **anywhere** counts. That last part was once trailing-only, on the theory that + "What time is it? It's 7:15." isn't awaiting a reply — true of that sentence + and wrong more often, since Bolt routinely asks and then keeps talking ("Want + me to fix it? I'd start with the config"), which is the case that actually + costs you a wake word. The asymmetry is the argument: an unwanted extra listen + ends itself on `VAD_GRACE_SECONDS` of silence, a missed one makes you start + over. `controller._should_follow_up` uses it to keep listening without the wake word, capped by `FOLLOW_UP_MAX_TURNS` so a server that ends every reply with a question can't loop forever off mic noise. Pure string logic, no Qt/audio imports. +- **`intents.py`** — the handful of utterances answered *without* the server. + "stop", "come here", "go to sleep", "say that again", "use your normal voice" + are commands to the body, and routing them through the desk API costs two to + four seconds and three network hops to make the pet walk left — and only works + if the server's prompt happens to advertise the matching `petctl` verb (which + is why `voice reset` needs a block in `ai/desk_api.py`'s pet prompt; see the + Voices section). Recognising the phrase here removes both the latency and that + coupling. The design problem is *not stealing real requests*, and three rules + cover it: whole-utterance exact match after normalisation (so "stop" is an + intent and "stop the docker container" is a question for Bolt), a closed table + with nothing arguable in it, and **never on a follow-up turn** — if Bolt just + asked you something your answer is his, and swallowing "never mind" locally + would leave the server holding a question it never got an answer to. Both + sides of the comparison go through `normalize()` (the table is canonicalised + at import, and `_build()` refuses to build one where two intents claim the + same normalised phrase, or where a phrase reduces to "" and would match pure + filler like "hey bolt"). Actions come back in the **same shape + `pet_actions.parse` produces**, so `PetWindow.apply_action` needs no new + vocabulary; the effects live in `controller._handle_local_intent`. Off switch: + `LOCAL_INTENTS=false`. - **`ui/`** — `app.py` wires `QApplication` + `PetWindow` + `PetTray` + the history/tuner windows + the push-to-talk hotkey + the controller thread together; `pet_window.py` is the frameless/translucent/always-on-top sprite @@ -417,6 +483,29 @@ the times it nearly heard you), and its slider is read per frame because `listen_for_wake_word` accepts a callable threshold. Set the threshold just under the peak you can hit reliably, then persist it in `.env`. +### The mic keeps recording while nothing is reading it + +Same family of bug as the openwakeword one above, one layer down: PortAudio +captures into a ring buffer continuously, so audio from a stretch where the +pipeline thread was busy elsewhere is still queued when the next read happens. +It bites in exactly one place. At the end of a reply that asked you something, +`_speak` sets `_talk_now` and the next turn starts recording immediately — with +the tail of the pet's own TTS sitting in that buffer, above the VAD threshold. +The VAD takes it for the start of your answer, Deepgram transcribes it, and Bolt +is handed his own last sentence as if you had said it. With barge-in on the +detector was draining the stream during playback so the window is small; with +`BARGE_IN=false` nothing drains it at all. + +`mic.flush(stream)` drops what's buffered, and `_speak` calls it on the +follow-up branch only. **That placement is the whole correctness argument** — +flushing is only safe where the buffer is known to hold nothing *you* said: +playback ran to completion, so if you had spoken, barge-in would have cut it and +taken the interrupted branch instead. Never flush before a wake-triggered +recording, where the rest of "thunderbolt, what time is it" is legitimately +queued and dropping it clips the request. A single call is bounded by +`max_seconds` so it can't chase a stream filling as fast as it drains, and it +no-ops on a stream with no `read_available` (i.e. every fake stream in tests). + ### Testing conventions `tests/` covers pure logic only (state machine, wake-word scoring loop, mic diff --git a/bolt_pet/audio/mic.py b/bolt_pet/audio/mic.py index 52beaaa..9bfb4ac 100644 --- a/bolt_pet/audio/mic.py +++ b/bolt_pet/audio/mic.py @@ -37,6 +37,42 @@ def rms(frame: np.ndarray) -> float: return float(np.sqrt(np.mean(frame.astype(np.float64) ** 2))) +def flush(stream, max_seconds: float = 10.0, sample_rate: int = config.SAMPLE_RATE) -> int: + """Throw away whatever is already sitting in the mic's buffer. Returns the + number of frames dropped. + + PortAudio keeps capturing into a ring buffer while nothing is reading it, so + audio recorded during a long blocking stretch is still queued when the next + read happens. That matters exactly once: at the end of a reply the pet is + about to listen for an answer, and the last fraction of a second of its own + TTS is in that buffer. It's above the VAD threshold, so `record_utterance` + treats it as the start of your answer, Deepgram transcribes it, and Bolt is + handed his own sentence as if you had said it. With barge-in on, the + detector was draining the stream during playback and the window is small; + with `BARGE_IN=false` nothing drains it at all. + + Only safe where the buffer is known to hold *nothing you said* — never + before a wake-triggered recording, where the rest of "thunderbolt, what + time is it" is legitimately queued and dropping it clips the request. + + *max_seconds* bounds a single call so this can't chase a stream that's + filling as fast as it's read. Best-effort: a fake stream in tests has no + `read_available` and this is a no-op, which is the correct behaviour for + one.""" + try: + available = int(getattr(stream, "read_available", 0) or 0) + except (TypeError, ValueError): + return 0 + if available <= 0: + return 0 + frames = min(available, int(max_seconds * sample_rate)) + try: + stream.read(frames) + except Exception: + return 0 # a mid-flush device error is the reader's problem, not ours + return frames + + def record_utterance( stream: AudioStream, should_continue=lambda: True, diff --git a/bolt_pet/config.py b/bolt_pet/config.py index ca1d62e..8ac4ec4 100644 --- a/bolt_pet/config.py +++ b/bolt_pet/config.py @@ -123,6 +123,14 @@ FOLLOW_UP_LISTEN = os.environ.get("FOLLOW_UP_LISTEN", "true").lower() in ("1", " FOLLOW_UP_MAX_TURNS = int(os.environ.get("FOLLOW_UP_MAX_TURNS", "10")) FOLLOW_UP_GRACE_SECONDS = float(os.environ.get("FOLLOW_UP_GRACE_SECONDS", "7")) +# ── local intents ─────────────────────────────────────────────────────────── +# A short, closed list of utterances the pet answers itself instead of paying a +# server round trip for: "stop", "come here", "go to sleep", "say that again", +# "use your normal voice". Matched whole and exact (see intents.py), never +# during a follow-up turn, so a real request is never swallowed. Turn it off to +# route absolutely everything through Bolt. +LOCAL_INTENTS = os.environ.get("LOCAL_INTENTS", "true").lower() in ("1", "true", "yes", "on") + COMMAND_TIMEOUT_SECONDS = int(os.environ.get("COMMAND_TIMEOUT_SECONDS", "30")) # ── sudo password prompts ─────────────────────────────────────────────────── @@ -221,6 +229,14 @@ NOTIFICATION_BRIDGE = os.environ.get("NOTIFICATION_BRIDGE", "false").lower() in # Regex matched against ": "; empty means "everything". NOTIFICATION_FILTER = os.environ.get("NOTIFICATION_FILTER", "") NOTIFICATION_MIN_INTERVAL_SECONDS = float(os.environ.get("NOTIFICATION_MIN_INTERVAL_SECONDS", "60")) +# Notifications arrive on the watcher thread and are forwarded from the +# heartbeat, which doesn't run while the pet is napping — so they queue. Both +# limits exist to stop an overnight backlog turning into a burst of round trips +# and a monologue at 8am: the queue is bounded (oldest dropped first) and +# anything staler than the age limit is discarded at drain time, because +# "Firefox finished downloading" is not news nine hours later. +NOTIFICATION_QUEUE_LIMIT = int(os.environ.get("NOTIFICATION_QUEUE_LIMIT", "20")) +NOTIFICATION_MAX_AGE_SECONDS = float(os.environ.get("NOTIFICATION_MAX_AGE_SECONDS", "900")) # ── file delivery ──────────────────────────────────────────────────────── # The server's deliver_files tool (ai/desk_api.py in the main tmn-api repo) diff --git a/bolt_pet/controller.py b/bolt_pet/controller.py index 9a840d3..fb5f235 100644 --- a/bolt_pet/controller.py +++ b/bolt_pet/controller.py @@ -15,15 +15,16 @@ from __future__ import annotations import threading import time +from collections import deque from typing import Optional from PySide6.QtCore import QObject, Signal from . import ( config, dialogue as dialogue_mod, file_delivery, file_ops, - history as history_mod, monitors as monitors_mod, notifications, - pet_actions, quiet, screen_context, screen_text, self_restart, - server_client, speech_text, updater, + history as history_mod, intents as intents_mod, monitors as monitors_mod, + notifications, pet_actions, quiet, screen_context, screen_text, + self_restart, server_client, speech_text, updater, ) from . import __version__ from .audio import barge_in, mic, stt, tts, wake_word @@ -98,7 +99,14 @@ class PetController(QObject): self._notification_gate = notifications.NotificationGate( config.NOTIFICATION_FILTER, config.NOTIFICATION_MIN_INTERVAL_SECONDS ) - self._pending_notifications: list[notifications.Notification] = [] + # Bounded, and stamped on arrival: the drain only runs from the + # heartbeat, which doesn't run while napping, so this fills up + # overnight. maxlen drops the oldest rather than growing without limit, + # and the stamp lets the drain discard a backlog nobody wants read out + # at 8am (see _drain_notifications). + self._pending_notifications: deque[tuple[float, notifications.Notification]] = deque( + maxlen=max(1, config.NOTIFICATION_QUEUE_LIMIT) + ) self._notification_lock = threading.Lock() # ── external controls (safe to call from the Qt/UI thread) ───────── @@ -187,18 +195,47 @@ class PetController(QObject): ) self.log.emit(f"Barge-in: {config.BARGE_IN_MODE} mode.") - with self._stream: - try: - health = server_client.check_health() - self.log.emit(f"Connected to server: {health}") - except Exception as exc: - self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.") - self._start_notification_bridge() - self._report_self_restart() - self._loop() - if self._notification_watcher is not None: - self._notification_watcher.stop() - self.finished.emit() + # Everything past here is in try/finally because `finished` is what + # ui/app.py waits on to quit the QThread and to run a pending + # os.execv. An exception escaping _loop used to skip it, leaving the + # thread wedged with the mic still open and no restart — so the failure + # mode of any bug below was "the pet goes deaf and the tray won't quit" + # rather than "one turn failed". + try: + with self._stream: + try: + health = server_client.check_health() + self.log.emit(f"Connected to server: {health}") + except Exception as exc: + self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.") + self._start_notification_bridge() + self._guarded(self._report_self_restart, "restart report") + self._loop() + except Exception as exc: + self.log.emit(f"Pipeline stopped unexpectedly: {exc!r}") + finally: + if self._notification_watcher is not None: + self._notification_watcher.stop() + self.finished.emit() + + def _guarded(self, work, label: str) -> bool: + """Run *work*, absorbing anything it raises. + + The pipeline is one thread driving a state machine that raises on an + illegal transition (deliberately — see state.py), plus a dozen + best-effort subsystems that shell out, hit the network, or touch the + filesystem. Any one of them raising something unforeseen used to end the + whole session. Here, it costs a log line and a forced return to IDLE, + which is the only state it's always safe to resume from. + + Returns True if *work* completed without raising.""" + try: + work() + return True + except Exception as exc: + self.log.emit(f"Recovered from a {label} failure: {exc!r}") + self._state.force(PetState.IDLE) + return False def _loop(self) -> None: while self._running: @@ -214,7 +251,7 @@ class PetController(QObject): if not self._running: return continue - self._handle_conversation_turn() + self._guarded(self._handle_conversation_turn, "conversation turn") def _wait_for_wake_or_click(self) -> bool: """True once either the wake phrase was heard or a click-to-talk @@ -226,7 +263,12 @@ class PetController(QObject): self._stream, should_continue=should_continue, threshold=self.wake_threshold, # callable: the tuner slider is live - on_tick=self._maybe_heartbeat, + # Guarded: on_tick is the one place control returns to us during a + # listen that can block for minutes, and everything it drives + # (update check, nap probe, notification forwarding) touches the + # network or shells out. Unguarded, any of them raising would unwind + # the listen loop and end the session. + on_tick=lambda: self._guarded(self._maybe_heartbeat, "heartbeat"), on_score=self._observe_wake_score, ) if not self._running: @@ -275,6 +317,13 @@ class PetController(QObject): self.log.emit(f"You: {text}") self.history.add(history_mod.USER, text, time.time()) + # "stop", "come here", "say that again" — answered here, without the + # round trip. Never on a follow-up turn: Bolt asked you something and + # the answer is his, even if it happens to look like a body command. + if not following_up and self._handle_local_intent(text): + self._state.transition(PetState.IDLE) + return + try: # What's focused right now rides along, so "what's this error?" # has a referent without you having to describe the window. @@ -293,6 +342,60 @@ class PetController(QObject): self._state.transition(PetState.IDLE) self._maybe_self_restart() + def _handle_local_intent(self, text: str) -> bool: + """Answer *text* locally if it's one of the closed set of body commands + in intents.py. Returns True if it was handled (no server call). + + The effects live here rather than in intents.py for the same reason + pet_actions splits parse from describe: recognising the phrase is pure + and testable, doing the thing needs the controller's state, the tray's + nap override and a Qt signal to the window.""" + if not config.LOCAL_INTENTS: + return False + intent = intents_mod.recognize(text) + if intent is None: + return False + self.log.emit(f"Local intent: {intent.name} (answered without the server)") + + if intent.name == "stop": + # Nothing to say and nothing to do: silence is the acknowledgement. + # Also ends any follow-up chain — "never mind" means the + # conversation is over, not that we should keep the mic open. + self._follow_ups = 0 + self._pending_follow_up = False + self._talk_now.clear() + return True + + if intent.name == "repeat": + last = self.history.last(history_mod.PET) + if last is None: + self._speak("I haven't said anything yet.") + else: + # remember=False: replaying a line isn't a new turn. Appending it + # would make "say that again" twice over read back as a + # conversation where Bolt volunteered the same thing three times. + self._speak(last.text, remember=False) + return True + + if intent.name == "voice_reset": + had_voice = bool(self._voice_id) + self.reset_voice() + self._speak(intent.speak if had_voice else "That is my normal voice.") + return True + + action = intent.action + if action is not None: + if action.get("action") == "nap": + # Through set_napping, not just the signal, so a spoken "go to + # sleep" overrides the quiet-hours schedule exactly like the + # tray's Nap entry and `petctl nap` do — otherwise the next + # schedule check would undo it within ten seconds. + self.set_napping(bool(action["enabled"])) + self.action.emit(dict(action)) + if intent.speak: + self._speak(intent.speak) + return True + def _with_context(self, text: str) -> str: """Everything the server gets alongside what you actually said: the focused window title, and a one-line note about the screen layout so @@ -319,9 +422,26 @@ class PetController(QObject): self._pet_monitor = int(index) def _handle_command(self, command: str) -> str: - """Server-relayed command. `petctl ...` drives the pet's body and - `filectl ...` does local file read/write/edit — neither ever reaches - a shell; everything else is a real command, exactly as before (see + """Server-relayed command, with the guarantee the relay depends on: this + always returns a string. + + The server is blocked on `/desk/tool_result` while this runs. If it + raises instead of answering, the relay never posts, the turn dies + mid-flight, and the server sits out its own timeout on a conversation it + can't finish — the worst available failure mode, because it's silent on + both ends. Handing the exception back as command output instead means + Bolt can read what went wrong and say so, or try something else, inside + the same turn.""" + try: + return self._dispatch_command(command) + except Exception as exc: + self.log.emit(f"Command handler failed: {exc!r}") + return f"[error] the pet couldn't run that: {exc}" + + def _dispatch_command(self, command: str) -> str: + """`petctl ...` drives the pet's body, `dialoguectl ...` plays a scene + and `filectl ...` does local file read/write/edit — none of them ever + reach a shell; everything else is a real command, exactly as before (see the security notes in the README).""" try: action = pet_actions.parse(command) @@ -568,14 +688,15 @@ class PetController(QObject): elif not config.VOICE_STICKY: self.reset_voice() - def _speak(self, text: str) -> None: + def _speak(self, text: str, remember: bool = True) -> None: self._state.transition(PetState.TALKING) # Bubble gets the markdown stripped but emoji kept (it can't render # **bold** but draws emoji fine); tts.speak() does its own, stricter # sanitizing for the voice. self.said.emit(speech_text.for_display(text)) self.log.emit(f"Bolt: {text}") - self.history.add(history_mod.PET, text, time.time()) + if remember: + self.history.add(history_mod.PET, text, time.time()) should_stop = None if self._barge_in is not None: @@ -608,6 +729,15 @@ class PetController(QObject): f"Asked a question — listening for your answer " f"({self._follow_ups}{'/' + str(cap) if cap > 0 else ''})." ) + # The tail of the reply we just played is still in the mic's ring + # buffer, and we're about to start recording with a VAD that will + # take it for the start of your answer — Bolt's own last words, + # transcribed and sent back to him as if you'd said them. Nothing you + # said can be in there: playback ran to completion, so if you had + # spoken, barge-in would have cut it and taken the other branch. + dropped = mic.flush(self._stream) + if dropped: + self.log.emit(f"Dropped {dropped} buffered frames of my own voice.") self._pending_follow_up = True self._talk_now.set() @@ -686,16 +816,39 @@ class PetController(QObject): def _queue_notification(self, notification: notifications.Notification) -> None: """Called on the watcher thread — just queue it; forwarding happens on the pipeline thread where it can't collide with a live conversation.""" - if not self._notification_gate.should_forward(notification, time.monotonic()): + now = time.monotonic() + if not self._notification_gate.should_forward(notification, now): return with self._notification_lock: - self._pending_notifications.append(notification) + if len(self._pending_notifications) == self._pending_notifications.maxlen: + # Say so rather than dropping in silence: a full queue means the + # bridge is matching more than the pet can plausibly speak, and + # the filter is what wants tightening. + self.log.emit("Notification queue full — dropping the oldest.") + self._pending_notifications.append((now, notification)) def _drain_notifications(self) -> None: with self._notification_lock: - pending, self._pending_notifications = self._pending_notifications, [] - for notification in pending: + pending = list(self._pending_notifications) + self._pending_notifications.clear() + + now = time.monotonic() + max_age = config.NOTIFICATION_MAX_AGE_SECONDS + if max_age > 0: + fresh = [entry for entry in pending if now - entry[0] <= max_age] + if len(fresh) != len(pending): + self.log.emit( + f"Skipping {len(pending) - len(fresh)} notification(s) older than " + f"{int(max_age)}s." + ) + pending = fresh + + for index, (_stamped, notification) in enumerate(pending): if not self._running or self._napping: + # Put back what we haven't forwarded — the old code swapped the + # queue out and then returned, silently dropping the remainder + # the moment a nap started mid-drain. + self._requeue_notifications(pending[index:]) return self.log.emit(f"Notification: {notification.as_text()}") self.history.add(history_mod.SYSTEM, notification.as_text(), time.time()) @@ -705,7 +858,11 @@ class PetController(QObject): on_command=self._handle_command, ) except server_client.ServerError as exc: + # Keep this one and everything behind it for the next heartbeat: + # the server being briefly down shouldn't silently eat the + # backlog. The age limit is what stops that retrying forever. self.log.emit(f"Couldn't forward notification: {exc}") + self._requeue_notifications(pending[index:]) return self._check_deliveries() self._apply_voice(reply) @@ -713,6 +870,15 @@ class PetController(QObject): self._speak(reply.text) self._state.transition(PetState.IDLE) + def _requeue_notifications(self, entries: list) -> None: + """Push undelivered notifications back on the front, oldest first, so a + retry keeps their original order (and their original timestamps, so a + retry loop can't keep a stale one alive indefinitely).""" + if not entries: + return + with self._notification_lock: + self._pending_notifications.extendleft(reversed(entries)) + # ── file delivery ──────────────────────────────────────────────────── def _check_deliveries(self) -> None: diff --git a/bolt_pet/intents.py b/bolt_pet/intents.py new file mode 100644 index 0000000..1f36b9f --- /dev/null +++ b/bolt_pet/intents.py @@ -0,0 +1,206 @@ +"""Things you say to the pet that the server has no business answering. + +"stop", "come here", "go to sleep", "say that again", "use your normal voice" — +none of these are questions for Bolt's brain. They're commands to the *body*, +and today every one of them costs a full turn: Deepgram, a `/desk/converse` +round trip, a model deciding to emit `petctl`, then ElevenLabs. Two to four +seconds and three network hops to make the pet walk left, and it only works at +all if the server's prompt happens to advertise the right verb — which is +exactly why `petctl voice reset` needs a block in the server's pet prompt (see +CLAUDE.md) or the model never emits it. Recognising the phrase here removes +both the latency and that coupling: "go back to your normal voice" works +whether or not the server was ever told the voice can be reset. + +The whole design problem is **not stealing real requests**. Three rules keep +it honest: + +1. **Whole-utterance, exact match after normalisation.** Never substring. So + "stop" is an intent and "stop the docker container" is a question for the + server — the distinction a substring match would destroy. +2. **The phrase table is closed and small.** Every entry is something with no + plausible reading as a request for Bolt to *do work*. Anything arguable + ("no thanks", "nothing") is deliberately absent — see rule 3 for why a + wrong guess is expensive. +3. **Nothing is recognised mid-conversation.** The controller skips this + entirely on a follow-up turn: if Bolt just asked you something, your answer + belongs to him, and swallowing "never mind" locally would leave the server + holding a question it never got an answer to. Local intents are only ever + for turns *you* started. + +Both sides of the comparison go through `normalize()` — the table is +canonicalised at import — so phrases can be written the way a person says them +("go back to your normal voice") without every variant having to be spelled +out. Filler is dropped from anywhere, not just the ends, because STT scatters +it ("hey bolt, could you please just stop now"). + +Pure classification, like pet_actions.parse: this module decides *what was +meant* and hands back an action in the same shape pet_actions produces, so +`controller.action` and `PetWindow.apply_action` need no new vocabulary. The +effects live in controller._handle_local_intent. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional + +# Words with no bearing on any command in the table, dropped wherever they +# appear. Kept deliberately short: every entry here is a word that can't +# distinguish one of these phrases from another, and adding one that can is how +# two intents quietly collide (the builder below raises if that happens). +_FILLER = frozenset({ + "the", "a", "an", "my", "your", "yours", "its", "to", "of", "and", + "please", "just", "that", "some", "bolt", "thunderbolt", "pet", "buddy", +}) + +# Dropped only from the front — the politeness/address ramp STT reliably +# prefixes. Not safe to drop mid-phrase (a bare "do" or "go" carries meaning +# elsewhere), which is why this is separate from _FILLER. +_LEADING_FILLER = frozenset({ + "hey", "hi", "hello", "yo", "ok", "okay", "um", "uh", "er", "so", + "can", "could", "would", "will", "you", "i", "id", "like", "lets", + "let", "us", "do", "go", "then", "now", +}) + +_TRAILING_FILLER = frozenset({ + "ok", "okay", "thanks", "thank", "you", "boy", "already", "now", +}) + +_KEEP = re.compile(r"[^a-z0-9 ]+") + + +def normalize(text: str) -> str: + """Reduce an utterance to the bare command, or "" if nothing is left. + + Lowercase, punctuation stripped (STT punctuates inconsistently), filler + dropped. Not a stemmer and deliberately not clever — its only job is to + make the same command spoken two ways land on the same string, without + ever turning one command into a different one.""" + words = [word for word in _KEEP.sub(" ", (text or "").lower()).split() + if word not in _FILLER] + while words and words[0] in _LEADING_FILLER: + words.pop(0) + while words and words[-1] in _TRAILING_FILLER: + words.pop() + return " ".join(words) + + +@dataclass(frozen=True) +class Intent: + """One recognised local command. + + *action* is a pet_actions-shaped dict for the UI (or None when there's + nothing for the body to do); *speak* is what to say out loud, empty for the + intents where doing the thing silently *is* the acknowledgement — the pet + visibly moves, and a spoken confirmation would only make it slower. "stop" + in particular has to be silent: answering "okay!" when told to be quiet is + a comedy sketch, not a feature. + """ + + name: str + action: Optional[dict] = None + speak: str = "" + + +# Intent -> (the Intent, the phrases that mean it, written as spoken). +_TABLE: tuple[tuple[Intent, tuple[str, ...]], ...] = ( + ( + Intent("stop"), + ("stop", "stop talking", "stop it", "be quiet", "quiet", "shut up", + "hush", "never mind", "nevermind", "forget it", "cancel", + "cancel that", "drop it", "enough"), + ), + ( + Intent("nap", {"action": "nap", "enabled": True}, "Night."), + ("go to sleep", "take a nap", "have a nap", "go to bed", "bedtime", + "goodnight", "good night", "get some rest"), + ), + ( + Intent("wake", {"action": "nap", "enabled": False}, "I'm up."), + ("wake up", "get up", "rise and shine", "you're awake", "are you awake"), + ), + ( + Intent("come", {"action": "move", "anchor": "cursor"}), + ("come here", "come to me", "come back", "over here", "follow me", + "follow my cursor"), + ), + ( + Intent("go_away", {"action": "move", "anchor": "bottom-right"}), + ("go away", "move over", "move out of the way", "get out of the way", + "out of the way", "hide", "get lost", "shoo", "scram", + "go somewhere else"), + ), + ( + Intent("repeat"), # answered from history by the controller + ("say that again", "say again", "repeat that", "repeat", + "what did you say", "what was that", "come again", "one more time", + "again", "sorry what"), + ), + ( + Intent("wander_on", {"action": "wander", "enabled": True}), + ("go for a walk", "wander", "wander around", "walk around", "explore", + "stretch your legs", "roam"), + ), + ( + Intent("wander_off", {"action": "wander", "enabled": False}), + ("stay still", "stay put", "stop moving", "stop wandering", + "don't move", "sit", "sit still", "stay", "settle down", "hold still"), + ), + ( + # Reachable from the server too (petctl voice reset), but only if its + # prompt mentions the verb. Recognising it here is what makes the + # phrase work regardless of what the server was told. + Intent("voice_reset", None, "Back to my own voice."), + ("use your normal voice", "use your own voice", "your normal voice", + "go back to your normal voice", "be yourself", "be yourself again", + "stop doing that voice", "drop the voice", "talk normally", + "speak normally", "use your real voice"), + ), +) + + +def _build() -> dict[str, Intent]: + """Canonicalise the table, refusing to build an ambiguous one. + + A phrase that normalises to "" would match an utterance of pure filler + ("hey bolt"), and one that lands on the same string as a phrase from + another intent would silently bind to whichever was declared last. Both are + edit-time mistakes, so they fail at import rather than at 3am on a mic.""" + table: dict[str, Intent] = {} + for intent, phrases in _TABLE: + for phrase in phrases: + key = normalize(phrase) + if not key: + raise ValueError(f"intent phrase {phrase!r} normalises to nothing") + existing = table.get(key) + if existing is not None and existing.name != intent.name: + raise ValueError( + f"phrase {phrase!r} ({key!r}) is claimed by both " + f"{existing.name} and {intent.name}" + ) + table[key] = intent + return table + + +_BY_PHRASE = _build() + +# Longest phrase in the table, in words. Anything longer can't match, so a real +# request skips normalisation entirely — this runs on every turn. +_MAX_WORDS = max(len(phrase.split()) for phrase in _BY_PHRASE) + + +def recognize(text: str) -> Optional[Intent]: + """The intent *text* expresses, or None to send it to the server. + + None is the safe answer and the common one: anything not matched verbatim + against the table belongs to Bolt.""" + raw = (text or "").strip() + if not raw: + return None + # +6 words of slack for the filler about to be stripped ("hey bolt, could + # you please stop" is six words to reach a one-word command). + if len(raw.split()) > _MAX_WORDS + 6: + return None + intent = _BY_PHRASE.get(normalize(raw)) + return intent diff --git a/bolt_pet/server_client.py b/bolt_pet/server_client.py index f0a60e1..1c1f1e2 100644 --- a/bolt_pet/server_client.py +++ b/bolt_pet/server_client.py @@ -19,6 +19,8 @@ Kept dependency-free beyond `requests` so it's easy to unit test with mocks. from __future__ import annotations +import os +import signal import subprocess from pathlib import Path from typing import Callable, NamedTuple, Optional @@ -29,6 +31,10 @@ from . import config, sudo_askpass _MAX_RELAY_HOPS = 16 +# Command output handed back up the relay is capped: it becomes part of the +# server's prompt, and a runaway `find /` would blow the context window. +_MAX_COMMAND_OUTPUT = 6000 + class ServerError(Exception): """Raised when the server responds with an error payload or unreachable.""" @@ -74,17 +80,61 @@ def run_local_command(command: str, timeout: int = None) -> str: timeout = timeout or config.SUDO_COMMAND_TIMEOUT_SECONDS timeout = timeout or config.COMMAND_TIMEOUT_SECONDS try: - completed = subprocess.run( - command, shell=True, capture_output=True, text=True, - timeout=timeout, cwd=str(Path.home()), env=env, + # start_new_session puts the shell in its own process group so a timeout + # can kill the whole tree. subprocess.run() would only SIGKILL the `sh` + # itself, leaving whatever it spawned (a build, a `tail -f`, an ffmpeg) + # running forever with no parent watching — one relayed command that + # hangs shouldn't leak a process for the rest of the session. + process = subprocess.Popen( + command, shell=True, cwd=str(Path.home()), env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=(os.name == "posix"), ) - output = (completed.stdout or "") + (completed.stderr or "") - return f"[exit {completed.returncode}]\n{output}"[:6000] - except subprocess.TimeoutExpired: - return f"[command timed out after {timeout}s]" except Exception as exc: return f"[command failed: {exc}]" + try: + stdout, stderr = process.communicate(timeout=timeout) + return _command_output(f"[exit {process.returncode}]", stdout, stderr) + except subprocess.TimeoutExpired: + stdout, stderr = _terminate(process) + # Whatever it managed to print before it hung is the useful part — a + # bare "timed out" tells the model nothing it can act on, and the last + # line of output usually says exactly what it was stuck waiting for. + return _command_output(f"[command timed out after {timeout}s]", stdout, stderr) + except Exception as exc: + _terminate(process) + return f"[command failed: {exc}]" + + +def _terminate(process: subprocess.Popen) -> tuple[str, str]: + """Kill a timed-out command's whole process group and collect what it wrote. + + SIGTERM first so a shell script can clean up, SIGKILL a moment later for + anything that ignores it. The final drain is itself time-boxed: a + grandchild holding the pipe open must not turn a timeout into a hang.""" + try: + if os.name == "posix": + group = os.getpgid(process.pid) + os.killpg(group, signal.SIGTERM) + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + os.killpg(group, signal.SIGKILL) + else: + process.kill() + except (ProcessLookupError, PermissionError, OSError): + pass # already gone, or never had its own group + try: + return process.communicate(timeout=2) + except Exception: + return "", "" + + +def _command_output(header: str, stdout: Optional[str], stderr: Optional[str]) -> str: + body = (stdout or "") + (stderr or "") + return f"{header}\n{body}"[:_MAX_COMMAND_OUTPUT] + def converse( text: str, @@ -132,6 +182,15 @@ def converse( voice_id=str(payload.get("voice_id") or ""), voice_name=str(payload.get("voice_name") or ""), ) + if payload.get("type") == "command": + # Fell out of the loop still being handed commands. Worth its own + # message: "unknown server response" sent everyone looking at the + # payload shape, when what actually happened is a model that kept + # calling tools and never answered. + raise ServerError( + f"the server kept relaying commands past the {_MAX_RELAY_HOPS}-hop cap " + "without producing a reply" + ) raise ServerError(str(payload.get("error") or "unknown server response")) diff --git a/bolt_pet/speech_text.py b/bolt_pet/speech_text.py index 8d161aa..ed9030d 100644 --- a/bolt_pet/speech_text.py +++ b/bolt_pet/speech_text.py @@ -67,6 +67,27 @@ _SPOKEN_SYMBOLS = { "=": " equals ", } +# Abbreviations a voice spells out letter by letter ("eee gee") because the +# periods make them look like sentence boundaries. Written out instead — this +# has to run before _UNSPEAKABLE strips anything, and the trailing \.? keeps +# "etc" working with or without its period. Word-bounded so "vs" inside a +# filename is left alone. +_SPOKEN_ABBREVIATIONS = ( + (re.compile(r"\be\.g\.?(?=\s|$)", re.IGNORECASE), "for example"), + (re.compile(r"\bi\.e\.?(?=\s|$)", re.IGNORECASE), "that is"), + (re.compile(r"\betc\.?(?=\s|$)", re.IGNORECASE), "and so on"), + (re.compile(r"\bvs\.?(?=\s|$)", re.IGNORECASE), "versus"), + (re.compile(r"\baka\b", re.IGNORECASE), "also known as"), + (re.compile(r"\bw/(?=\s)", re.IGNORECASE), "with"), + # "PR #42" -> "PR number 42"; a bare "#" is markup and _UNSPEAKABLE drops it. + (re.compile(r"#(?=\d)"), "number "), + # A long option's dashes are punctuation to the eye and syllables to the ear + # ("dash dash force"). Only the doubled form: a single hyphen has to survive + # for "bolt-pet" and "up-to-date", and requiring a word character after it + # keeps a "---" horizontal rule intact for _RULE to strip. + (re.compile(r"(? str: return "" for symbol, spoken in _PRE_SPOKEN_SYMBOLS.items(): text = text.replace(symbol, spoken) + # Before the markdown pass, so "#42" still has its "#" to word and a real + # "## Heading" (no digit after the hashes) is left for _HEADING to strip. + for pattern, spoken in _SPOKEN_ABBREVIATIONS: + text = pattern.sub(spoken, text) text = _strip_markdown(text, keep_emoji=False) text = _URL.sub(" link ", text) text = _TABLE_PIPE.sub(", ", text) @@ -119,17 +144,21 @@ def for_speech(text: str) -> str: def is_question(text: str) -> bool: - """True if the spoken reply contains a question mark — the cue for the - pet to keep listening instead of making you say the wake word again. + """True if the reply asks the user anything — the cue for the pet to keep + listening instead of making you say the wake word again. - The check runs on the spoken form, so a '?' that only exists inside a - stripped code block or a URL doesn't count, and trailing decoration - (emoji, quotes, brackets) is peeled off first so "Ready to go? 🚀" - still reads as a question.""" - spoken = for_speech(text) - while spoken and not (spoken[-1].isalnum() or spoken[-1] in "?."): - spoken = spoken[:-1] - return "?" in spoken + Anywhere in the reply counts, not only the end. An earlier version required + a *trailing* '?' on the theory that "What time is it? It's 7:15." isn't + waiting on an answer, and that's true of that sentence but wrong far more + often: Bolt routinely asks first and then keeps talking ("Want me to fix + it? I'd start with the config."), and refusing to listen there is the case + that actually costs you a wake word. The cheap failure is the other + direction — an unwanted extra listen ends itself on `VAD_GRACE_SECONDS` of + silence, and `FOLLOW_UP_MAX_TURNS` caps the chain. + + The test runs on the *spoken* form, so a '?' that only exists inside a + stripped code block, a URL, or a markdown link target doesn't count.""" + return "?" in for_speech(text) def for_display(text: str) -> str: diff --git a/tests/test_controller_features.py b/tests/test_controller_features.py index f76c674..8f1a9a3 100644 --- a/tests/test_controller_features.py +++ b/tests/test_controller_features.py @@ -221,10 +221,13 @@ def test_a_statement_does_not_keep_listening(spoke, ctrl): assert not ctrl._pending_follow_up -def test_a_question_in_passing_does_not_count(spoke, ctrl): - """Only a reply that *ends* on a question is waiting for an answer.""" - ctrl._speak("What time is it? It's 7:15 AM.") - assert not ctrl._talk_now.is_set() +def test_a_question_anywhere_in_the_reply_keeps_listening(spoke, ctrl): + """Bolt often asks and then keeps talking ("Want me to fix it? I'd start + with the config."), so the question mark doesn't have to be last. An + unwanted extra listen ends itself on VAD_GRACE_SECONDS of silence; a missed + one costs you a wake word, which is the more expensive mistake.""" + ctrl._speak("Want me to restart it? It's been up for 40 days.") + assert ctrl._talk_now.is_set() def test_follow_ups_stop_at_the_cap(spoke, monkeypatch, ctrl): @@ -382,6 +385,148 @@ def test_napping_still_answers_when_spoken_to(monkeypatch, ctrl): assert spoken == ["always"] +# ── local intents ─────────────────────────────────────────────────────────── + +@pytest.fixture +def heard(monkeypatch): + """A turn where you said something, with the server and TTS recorded. + Returns (utterance_setter, sent, spoken).""" + said = {"text": ""} + sent, spoken = [], [] + monkeypatch.setattr(controller_mod.mic, "record_utterance", + lambda *a, **k: np.zeros(10, dtype=np.int16)) + monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: said["text"]) + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None: sent.append(text) or Reply("from the server")) + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None: + spoken.append(text) or True) + return said, sent, spoken + + +def test_a_local_intent_never_reaches_the_server(heard, ctrl): + said, sent, spoken = heard + said["text"] = "come here" + actions = _capture(ctrl.action) + + ctrl._handle_conversation_turn() + + assert sent == [] # no round trip at all + assert actions == [{"action": "move", "anchor": "cursor"}] + assert spoken == [] # walking over is the reply + assert ctrl._state.state == PetState.IDLE + + +def test_stop_is_answered_with_silence(heard, ctrl): + said, sent, spoken = heard + said["text"] = "be quiet" + ctrl._handle_conversation_turn() + assert (sent, spoken) == ([], []) + + +def test_a_request_that_merely_starts_with_an_intent_word_goes_to_the_server(heard, ctrl): + said, sent, spoken = heard + said["text"] = "stop the docker container" + ctrl._handle_conversation_turn() + assert sent and "stop the docker container" in sent[0] + assert spoken == ["from the server"] + + +def test_local_intents_are_skipped_while_answering_a_question(heard, ctrl): + """Bolt asked something; "never mind" is an answer to him, not a body + command. Swallowing it locally would leave the server holding a question it + never got a reply to.""" + said, sent, spoken = heard + said["text"] = "never mind" + ctrl._pending_follow_up = True + + ctrl._handle_conversation_turn() + + assert sent and "never mind" in sent[0] + + +def test_local_intents_can_be_turned_off(monkeypatch, heard, ctrl): + monkeypatch.setattr(controller_mod.config, "LOCAL_INTENTS", False) + said, sent, spoken = heard + said["text"] = "come here" + ctrl._handle_conversation_turn() + assert sent and "come here" in sent[0] + + +def test_say_that_again_replays_the_last_line_without_duplicating_history(heard, ctrl): + said, sent, spoken = heard + ctrl.history.add(controller_mod.history_mod.PET, "it's 7:15 AM", 0.0) + said["text"] = "what did you say?" + + ctrl._handle_conversation_turn() + + assert spoken == ["it's 7:15 AM"] + assert sent == [] + pet_lines = [e.text for e in ctrl.history.entries() + if e.role == controller_mod.history_mod.PET] + assert pet_lines == ["it's 7:15 AM"] # replayed, not re-recorded + + +def test_repeat_with_nothing_to_repeat_says_so(heard, ctrl): + said, sent, spoken = heard + said["text"] = "say that again" + ctrl._handle_conversation_turn() + assert spoken == ["I haven't said anything yet."] + + +def test_going_back_to_the_normal_voice_needs_no_server_prompt_support(heard, ctrl): + """The server can only offer `petctl voice reset` if its prompt happens to + advertise the verb; recognising the phrase here works regardless.""" + said, sent, spoken = heard + ctrl._voice_id, ctrl._voice_name = "voice-123", "Brian" + changed = _capture(ctrl.voice_changed) + said["text"] = "go back to your normal voice" + + ctrl._handle_conversation_turn() + + assert ctrl._voice_id == "" + assert changed == [""] + assert spoken == ["Back to my own voice."] + assert sent == [] + + +def test_go_to_sleep_overrides_the_quiet_hours_schedule(heard, ctrl): + said, sent, spoken = heard + said["text"] = "go to sleep" + + ctrl._handle_conversation_turn() + + assert ctrl._napping is True + assert ctrl._nap_forced is True # not undone by the next schedule check + assert spoken == ["Night."] + + +# ── failure containment ───────────────────────────────────────────────────── + +def test_one_bad_turn_does_not_end_the_session(monkeypatch, ctrl): + """A turn raising something unforeseen used to unwind _loop and kill the + thread — the pet would go deaf until it was restarted by hand.""" + logs = _capture(ctrl.log) + + def explode(): + raise RuntimeError("numpy said no") + + assert ctrl._guarded(explode, "conversation turn") is False + assert ctrl._state.state == PetState.IDLE + assert any("Recovered from a conversation turn failure" in m for m in logs) + + +def test_a_command_handler_crash_is_reported_up_the_relay(monkeypatch, ctrl): + """The server is blocked on /desk/tool_result while this runs. Raising would + leave it waiting out its own timeout on a turn that can never finish.""" + monkeypatch.setattr(controller_mod.pet_actions, "parse", + lambda command: (_ for _ in ()).throw(KeyError("boom"))) + + output = ctrl._handle_command("petctl move top-left") + + assert output.startswith("[error]") and "boom" in output + + # ── notification bridge ───────────────────────────────────────────────────── def test_notifications_are_forwarded_and_spoken(monkeypatch, ctrl): @@ -404,7 +549,54 @@ def test_notifications_are_forwarded_and_spoken(monkeypatch, ctrl): def test_filtered_out_notifications_are_never_queued(ctrl): ctrl._notification_gate = controller_mod.notifications.NotificationGate("deploy", 0) ctrl._queue_notification(Notification(app="Chat", summary="lunch?", body="")) - assert ctrl._pending_notifications == [] + assert not ctrl._pending_notifications + + +def test_the_notification_queue_is_bounded(monkeypatch, ctrl): + """An overnight nap can't grow the queue without limit — the drain only runs + from the heartbeat, and the heartbeat doesn't run while napping.""" + monkeypatch.setattr(controller_mod.config, "NOTIFICATION_QUEUE_LIMIT", 3) + ctrl = controller_mod.PetController() + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + + for index in range(10): + ctrl._queue_notification(Notification(app="CI", summary=f"build {index}", body="")) + + queued = [notification.summary for _stamp, notification in ctrl._pending_notifications] + assert queued == ["build 7", "build 8", "build 9"] # oldest dropped + + +def test_stale_notifications_are_dropped_instead_of_read_out(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "NOTIFICATION_MAX_AGE_SECONDS", 900) + sent = [] + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None: sent.append(text) or Reply("")) + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + ctrl._queue_notification(Notification(app="CI", summary="fresh", body="")) + # Backdate it past the age limit, as an overnight backlog would be. + stamp, notification = ctrl._pending_notifications.pop() + ctrl._pending_notifications.append((stamp - 4000, notification)) + + ctrl._drain_notifications() + + assert sent == [] + + +def test_a_nap_starting_mid_drain_keeps_the_rest_queued(monkeypatch, ctrl): + """The old code swapped the queue out and returned, losing the remainder.""" + def converse(text, on_command=None): + ctrl._napping = True # e.g. quiet hours began, or a fullscreen app opened + return Reply("") + + monkeypatch.setattr(controller_mod.server_client, "converse", converse) + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + for index in range(3): + ctrl._queue_notification(Notification(app="CI", summary=f"build {index}", body="")) + + ctrl._drain_notifications() + + remaining = [notification.summary for _stamp, notification in ctrl._pending_notifications] + assert remaining == ["build 1", "build 2"] def test_notifications_are_not_forwarded_while_napping(monkeypatch, ctrl): diff --git a/tests/test_intents.py b/tests/test_intents.py new file mode 100644 index 0000000..4830da5 --- /dev/null +++ b/tests/test_intents.py @@ -0,0 +1,112 @@ +"""Local intent recognition — pure string logic, no hardware or display. + +The interesting tests are the negative ones: this feature's whole risk is +swallowing something that was meant for the server. +""" + +import sys +from pathlib import Path + +import pytest + +from bolt_pet import intents + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + + +@pytest.mark.parametrize("said, expected", [ + ("stop", "stop"), + ("Stop!", "stop"), + ("never mind", "stop"), + ("be quiet", "stop"), + ("go to sleep", "nap"), + ("take a nap", "nap"), + ("goodnight", "nap"), + ("wake up", "wake"), + ("come here", "come"), + ("follow my cursor", "come"), + ("get out of the way", "go_away"), + ("hide", "go_away"), + ("say that again", "repeat"), + ("what did you say?", "repeat"), + ("go for a walk", "wander_on"), + ("stay put", "wander_off"), + ("sit", "wander_off"), + ("use your normal voice", "voice_reset"), + ("go back to your normal voice", "voice_reset"), + ("be yourself again", "voice_reset"), +]) +def test_recognized_phrases(said, expected): + intent = intents.recognize(said) + assert intent is not None and intent.name == expected + + +@pytest.mark.parametrize("said", [ + # Each of these starts with (or contains) an intent phrase, and every one is + # a real request. A substring match would eat all of them. + "stop the docker container", + "stop the deploy and tell me what broke", + "can you hide the window that's covering my terminal", + "come up with a name for this branch", + "repeat the last command but with sudo", + "what did you say the disk usage was on the server", + "move the config file to the backup directory", + "sit down and write me a haiku about kubernetes", + "what time is it", + "go to sleep mode on the server", + "", + " ", + # Pure filler leaves an empty string, which must not match anything. + "hey bolt", + "okay bolt please", +]) +def test_real_requests_are_left_for_the_server(said): + assert intents.recognize(said) is None + + +def test_filler_is_stripped_from_both_ends(): + assert intents.normalize("Hey Bolt, could you please just stop now?") == "stop" + assert intents.normalize("okay, come here buddy") == "come here" + + +def test_normalize_returns_empty_for_pure_filler(): + assert intents.normalize("hey bolt") == "" + assert intents.normalize("...") == "" + + +def test_intents_carry_ui_actions_in_the_pet_actions_shape(): + """The action dicts go straight to PetWindow.apply_action, so they have to + match the vocabulary pet_actions.parse produces — no new UI cases.""" + assert intents.recognize("come here").action == {"action": "move", "anchor": "cursor"} + assert intents.recognize("stay put").action == {"action": "wander", "enabled": False} + assert intents.recognize("go to sleep").action == {"action": "nap", "enabled": True} + + +def test_stop_says_nothing(): + """Answering "okay!" when told to be quiet defeats the purpose.""" + intent = intents.recognize("be quiet") + assert intent.speak == "" and intent.action is None + + +def test_a_phrase_claimed_by_two_intents_fails_at_import(monkeypatch): + """Without this guard the phrase would silently bind to whichever intent was + declared last — a table edit that looks fine and misbehaves on a mic.""" + monkeypatch.setattr(intents, "_TABLE", ( + (intents.Intent("stop"), ("enough",)), + (intents.Intent("nap"), ("enough",)), + )) + with pytest.raises(ValueError, match="claimed by both"): + intents._build() + + +def test_a_phrase_of_pure_filler_fails_at_import(monkeypatch): + """It would normalise to "" and then match any all-filler utterance.""" + monkeypatch.setattr(intents, "_TABLE", ((intents.Intent("stop"), ("please bolt",)),)) + with pytest.raises(ValueError, match="normalises to nothing"): + intents._build() + + +def test_every_table_phrase_round_trips(): + for phrase, intent in intents._BY_PHRASE.items(): + assert phrase, "a phrase normalised to nothing" + assert intents.recognize(phrase) is intent diff --git a/tests/test_mic.py b/tests/test_mic.py index f20a9de..f4ee903 100644 --- a/tests/test_mic.py +++ b/tests/test_mic.py @@ -112,3 +112,51 @@ def test_pcm_to_wav_bytes_round_trips_via_wave_module(): assert wf.getframerate() == 16000 frames = wf.readframes(wf.getnframes()) assert np.frombuffer(frames, dtype=np.int16).tolist() == pcm.tolist() + + +# ── flushing buffered audio ───────────────────────────────────────────────── + +class _BufferedStream: + """A stream with a backlog, like PortAudio's ring buffer after the reader + was blocked on a network call for a while.""" + + def __init__(self, available): + self.read_available = available + self.reads = [] + + def read(self, frames): + self.reads.append(frames) + self.read_available = max(0, self.read_available - frames) + return np.zeros((frames, 1), dtype=np.int16), False + + +def test_flush_drops_exactly_what_was_buffered(): + stream = _BufferedStream(4096) + assert mic.flush(stream) == 4096 + assert stream.reads == [4096] + assert stream.read_available == 0 + + +def test_flush_is_bounded_so_it_cannot_chase_a_live_stream(): + """A stream filling as fast as it drains must not spin forever.""" + stream = _BufferedStream(10 ** 9) + dropped = mic.flush(stream, max_seconds=1.0, sample_rate=16000) + assert dropped == 16000 + + +def test_flush_is_a_noop_on_an_empty_or_fake_stream(): + stream = _BufferedStream(0) + assert mic.flush(stream) == 0 + assert stream.reads == [] + assert mic.flush(_ScriptedStream([])) == 0 # no read_available at all + assert mic.flush(None) == 0 + + +def test_flush_swallows_a_device_error(): + class _Broken: + read_available = 1024 + + def read(self, frames): + raise RuntimeError("device disappeared") + + assert mic.flush(_Broken()) == 0 diff --git a/tests/test_server_client.py b/tests/test_server_client.py index 7c3b8a7..4b18664 100644 --- a/tests/test_server_client.py +++ b/tests/test_server_client.py @@ -1,4 +1,6 @@ +import os import sys +import time from pathlib import Path from unittest.mock import MagicMock, patch @@ -142,3 +144,52 @@ def test_download_outbox_file_raises_server_error_on_http_failure(): get.return_value = _mock_response({}, ok=False) with pytest.raises(server_client.ServerError, match="abc"): server_client.download_outbox_file("abc") + + +def test_converse_reports_a_relay_that_never_produced_a_reply(): + """Hitting the hop cap used to surface as "unknown server response", which + sent everyone looking at the payload shape instead of at a model that kept + calling tools and never answered.""" + command = {"type": "command", "command": "echo hi", "token": "t"} + with patch.object(server_client.requests, "post") as post: + post.return_value = _mock_response(command) + with pytest.raises(server_client.ServerError, match="hop cap"): + server_client.converse("hi", on_command=lambda cmd: "ok") + + +# ── relayed shell commands ────────────────────────────────────────────────── + +@pytest.fixture(autouse=True) +def _no_sudo_prompt(monkeypatch): + monkeypatch.setattr(server_client.config, "SUDO_ASKPASS_PROMPT", False) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX shell and process groups") +def test_a_successful_command_returns_its_output_and_exit_code(): + output = server_client.run_local_command("echo hello; exit 3") + assert output.startswith("[exit 3]") + assert "hello" in output + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX shell and process groups") +def test_a_timed_out_command_still_reports_what_it_printed(): + """A bare "timed out" tells the model nothing; the last line of output + usually says exactly what it was stuck waiting for.""" + output = server_client.run_local_command("echo working on it; sleep 30", timeout=1) + assert "timed out after 1s" in output + assert "working on it" in output + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX shell and process groups") +def test_a_timed_out_command_takes_its_children_with_it(tmp_path): + """subprocess.run() would only kill the `sh`, leaving whatever it spawned + running for the rest of the session with no parent watching.""" + marker = tmp_path / "ticks" + server_client.run_local_command( + f"(while true; do echo tick >> {marker}; sleep 0.05; done) & sleep 30", + timeout=1, + ) + settled = marker.stat().st_size if marker.exists() else 0 + time.sleep(0.4) + grew = (marker.stat().st_size if marker.exists() else 0) - settled + assert grew == 0, "a grandchild survived the timeout and is still writing" diff --git a/tests/test_speech_text.py b/tests/test_speech_text.py index 6bfbd7f..b3f2276 100644 --- a/tests/test_speech_text.py +++ b/tests/test_speech_text.py @@ -78,3 +78,29 @@ def test_is_question_ignores_question_marks_that_are_not_spoken(): assert not is_question("Docs are at https://example.com/x?y=1") assert not is_question("") assert not is_question(None) + + +def test_abbreviations_are_worded_instead_of_spelled_out(): + """The periods make these look like sentence boundaries, so the voice reads + them letter by letter ("eee gee").""" + assert for_speech("Use a flag, e.g. --force") == "Use a flag, for example force" + assert for_speech("i.e. the config file") == "that is the config file" + assert for_speech("logs, configs, etc.") == "logs, configs, and so on" + assert for_speech("docker vs. podman") == "docker versus podman" + assert for_speech("Fixed in PR #42") == "Fixed in PR number 42" + + +def test_abbreviation_wording_is_word_bounded(): + """"vs" inside a word or filename isn't an abbreviation.""" + assert "versus" not in for_speech("the vscode window") + assert "versus" not in for_speech("revs per minute") + # A markdown heading has no digit after the hashes, so it's still a heading. + assert for_speech("## Results") == "Results" + + +def test_long_option_dashes_are_dropped_but_hyphens_survive(): + assert for_speech("run it with --force") == "run it with force" + assert for_speech("check bolt-pet is up-to-date") == "check bolt-pet is up-to-date" + # The rule line is gone; the full stops are _bullets_to_sentences giving the + # voice a pause where the eye saw a line break. + assert for_speech("one\n---\ntwo") == "one. two."