diff --git a/.env.example b/.env.example index c0da76b..e861303 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,9 @@ ELEVENLABS_VOICE_ID= #PET_CLICK_THROUGH=false #PET_EDGE_SNAP=true #PET_SNAP_MARGIN=48 +# Start where you last dragged it. A position on a monitor that's no longer +# connected is ignored, so unplugging a screen can't hide the pet off-desktop. +#PET_REMEMBER_POSITION=true # ── Barge-in (optional) — interrupt the pet mid-sentence ──────────────────── # BARGE_IN_MODE decides what counts as an interruption: @@ -138,6 +141,16 @@ ELEVENLABS_VOICE_ID= # ── Streaming TTS (optional) — starts talking on the first chunk ──────────── #TTS_STREAMING=true +# ── Latency: streaming the reply and the transcript ───────────────────────── +# STREAMING_REPLIES speaks each sentence as the server generates it, instead of +# waiting out the whole model call before the first word. STT_STREAMING sends +# mic frames to Deepgram as you talk, so the transcript is ready the moment you +# stop. Both fall back to the old path automatically if anything goes wrong. +# VAD_SILENCE_END_SEC is the other half: it is dead air on every single turn, +# so 0.8-1.0 feels markedly snappier than the 1.2 default. +#STREAMING_REPLIES=true +#STT_STREAMING=true + # ── Screen context (optional) ─────────────────────────────────────────────── # Sends the focused window's title along with what you said, so "what's this # error?" has a referent. Text only — no screenshots leave the machine. diff --git a/CLAUDE.md b/CLAUDE.md index 626a498..4b73088 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,10 @@ all suppressed while it's napping (quiet hours / fullscreen DND). ./run.sh # macOS/Linux run.bat # Windows +# Preflight: is this install actually going to work? (config, mic, keys, sprites…) +python -m bolt_pet --doctor # shallow: no network, no mic +python -m bolt_pet --doctor --deep # contacts the server and opens the microphone + # Run tests (no pytest config file — tests self-insert repo root via sys.path). # QT_QPA_PLATFORM=offscreen avoids a QApplication segfault on headless/no-display hosts. QT_QPA_PLATFORM=offscreen .venv/bin/pytest tests/ @@ -239,7 +243,18 @@ logs a missing-config message and exits its thread instead of starting. - **`quiet.py`** — quiet-hours spec parsing (`23:00-08:00`, wraps midnight, comma-separated). Napping suppresses *proactive* noise and wandering only; wake word / click / push-to-talk still work. -- **`notifications.py`** — Linux/D-Bus notification bridge: tails +- **`notifications.py`** — Linux/D-Bus notification bridge. Two latency/loss bugs + fixed 2026-08-02, both in `controller._maybe_heartbeat`: draining was wired to the + 60s heartbeat interval rather than the ~1.2s wake tick, and a heartbeat that landed + mid-conversation stamped its own clock *before* checking — burning the slot and + waiting another full interval, repeatedly, which is how a notification could go + unspoken for five or ten minutes. Draining now runs on every tick while IDLE, and the + heartbeat clock only advances when the heartbeat actually runs. Separately, the rate + limit used to *drop* notifications inside its window (a second text a minute later was + silently lost); the filter still gates at queue time but the limit is gone — a burst + is **batched into one turn** instead, same single round trip, no lost messages, capped + by `_MAX_PENDING_NOTIFICATIONS`. +- **`notifications.py` internals** — tails `dbus-monitor`, parses Notify calls (pure `iter_notifications()`), filters and rate-limits them (`NotificationGate`), and the controller forwards survivors through `converse()`. Off by default — each one is a round trip. @@ -275,6 +290,81 @@ logs a missing-config message and exits its thread instead of starting. - **`hotkey.py`** — global push-to-talk via `pynput`; soft-fails with a logged reason (Wayland, missing package, macOS permissions) since the wake word is the primary trigger. +- **`audio/stt_stream.py`** — streaming speech-to-text. The one-shot path waits for + the utterance to end, uploads the whole WAV, then waits again; that second wait is + dead time that grows with how long you spoke. Deepgram's live websocket removes it: + `record_utterance(on_frame=...)` hands each captured frame to a + `StreamingTranscriber`, so by the time the VAD decides you stopped the transcript is + essentially already there. Three deliberate limits: `open()` returning **None is an + ordinary outcome** (no websocket-client, no network, no key) because the full audio is + still buffered and `controller._transcribe` just falls back; the **local VAD still + decides when you stopped** rather than Deepgram's endpointing, since barge-in, + follow-up listening and the grace period are all built on it and coupling them to the + network is not a first-pass change; and the socket is **per-utterance**, because + holding one open across an idle pet bills for silence and dies on the first blip. + Off: `STT_STREAMING=false`. +- **Streamed replies** — `server_client.converse_stream()` reads NDJSON from the desk + API's `/desk/converse_stream` and speaks each sentence as it arrives + (`controller._speak_stream_chunk`), so the wait is time-to-first-sentence instead of + the whole model call. `Reply.spoken` marks a reply whose sentences were already said: + the text is still carried, because the follow-up rule needs to see whether it ended on + a question, but it must not be read out again. A stream that fails *before* anything + was spoken falls back to `converse()` invisibly; one that fails after ends the turn + quietly rather than repeating the first half. Off: `STREAMING_REPLIES=false`. +- **`speech.py`** — everything the pet says, and the policy differences between + kinds of saying. There were four near-copies of this in `controller.py` (a + reply, a holding line, a streamed sentence, a dialogue scene), each repeating + the same dance — transition state, show the bubble, maybe record history, + reset barge-in, call TTS, reset barge-in *again*, resume state, decide + whether to keep the mic open — and they had already drifted apart: one forgot + to arm barge-in, another skipped the follow-up rule. Now the dance is + `Speaker.say()` and the differences are data on a frozen `Utterance` + (`record`, `resume`, `hold_talking`, `follow_up`, `interruptible`), built by + the four classmethods `reply`/`holding`/`stream_chunk`/`scene`. Notable + policies: a holding line is **not** recorded (it's filler; the transcript + should keep the answer) and **not** interruptible (cutting off "give me a + sec" strands the tool already running), and it resumes the state it + interrupted rather than dropping to IDLE, because the turn isn't over. A + streamed chunk stays TALKING so the sprite doesn't flicker between sentences. + Collaborators are injected, so all of this is tested without Qt, audio or a + real state machine. Two subtleties that are bugs waiting to happen: the + barge-in detector is read through a **callable**, not held (it's built after + the Speaker — it needs the mic stream — and swapped when the mode changes; two + copies drifting apart is invisible until the wake model starts hearing the + pet), and `last_detail` is captured *before* the post-playback reset, since + reading it after means every interruption reports zeroed counters. + `follow_up_decision()` is the mic-open rule as a pure function. +- **Lip-sync** — the mouth is driven by the audio, not a timer. + `audio/tts.level_of(frame)` reduces a PCM frame to 0..1 loudness on a **sqrt + curve** (speech sits well below peak most of the time, so a linear map leaves + the mouth barely open during normal talking), `envelope()` does the same for a + whole clip, and playback calls the `on_level` hook per chunk. That travels + `Speaker` → `controller.mouth` (a Signal) → `PetWindow.set_mouth`, and + `_mouth_frame()` indexes the talking frames directly — which works because + `generate_bolt_sprites.py` draws them as an **openness ramp** (closed first, + widest last) rather than an arbitrary loop. Levels going stale + (`_MOUTH_STALE_SECONDS`) hands control back to the ordinary animation, so + offline TTS — which has no envelope — degrades to the timed loop instead of + freezing the mouth mid-syllable. +- **`window_state.py`** — where the pet was left, so it starts there. + `~/.cache/bolt-pet/window.json`, atomic write on drag-end, every function + swallows its own errors (a corrupt state file must mean the default corner, + never a pet that won't start). `is_visible_on()` re-validates against the + *current* screen layout on load, because the common case for a stale position + is exactly the dangerous one: the pet was last on a monitor that is now + unplugged, and restoring it faithfully puts it somewhere unreachable. Takes + plain rectangles rather than importing Qt. Off: `PET_REMEMBER_POSITION=false`. +- **`doctor.py`** — `python -m bolt_pet --doctor`, a preflight, written after a + week of debugging things one command would have shown: a venv whose python + was a zero-byte file, an OCR engine never installed, a wrong proxy header. + Twelve independent checks, each reporting ok / warn (degraded but working) / + fail, and each saying **what to do about it** — `screen reading: warn` is + useless alone, `apt install tesseract-ocr` is the whole point. Nothing raises: + a doctor that crashes on a broken install is diagnosing the wrong patient, so + `run()` catches per-check and a failed check becomes a FAIL row rather than a + traceback. Shallow by default (no network, no mic) since it's the first thing + you reach for when the network is what's broken; `--deep` actually contacts + the server and opens the microphone. - **`speech_text.py`** — sanitizes server replies before they're heard/shown. `for_speech()` (called inside `tts.speak()`, so every path to the speakers is covered) strips markdown, emoji, URLs and stray symbols the voice would read @@ -284,7 +374,7 @@ logs a missing-config message and exits its thread instead of starting. whether a reply leaves the pet waiting on an answer: it tests the *spoken* form (so a '?' inside a stripped code block or URL doesn't count) and only a trailing one counts, since a question asked in passing isn't awaiting a - reply. `controller._should_follow_up` uses it to keep listening without the + reply. `speech.follow_up_decision` uses it to keep listening without the wake word, capped by `FOLLOW_UP_MAX_TURNS` so a server that ends every reply with a question can't loop forever off mic noise. Pure string logic, no Qt/audio imports. @@ -414,6 +504,20 @@ HTTP chunk reassembly by `chunks_to_int16`, emote motion by `screen_context.context_for` / `is_fullscreen_active`, otherwise they shell out to xprop on a headless box. +**`test_pipeline_smoke.py` is the exception, deliberately.** Unit tests inject +a fake at the seam they care about, and a run of production bugs — notifications +sitting unspoken for minutes, the pet saying things twice, `[laughing]` read +aloud, a device command arriving as prose — got through with every one of them +green, because each was an *interaction* between two individually-correct +units. So that file stands up a real threaded HTTP server on a loopback port, +speaks the desk protocol at it, and drives whole turns through the real +`server_client` (including NDJSON streaming), the real controller and the real +state machine, faking only the mic stream and the speakers. When a bug crosses +a module boundary, add the case there; when it lives inside one module, the +pure-function pattern above is still the cheaper test. It is also worth +mutation-checking a new case — break the source line it is meant to catch and +confirm it actually goes red. + ## Security notes The server can relay a shell command back to this machine to execute as the diff --git a/README.md b/README.md index 6f206ab..af62007 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,33 @@ near misses — frames that scored just under the threshold — and the slider takes effect immediately, mid-listen. Set the threshold just below the peak you can hit reliably, then write it into `.env` as `WAKE_WORD_THRESHOLD`. +## Something not working? + +```bash +python -m bolt_pet --doctor +``` + +Checks the things that make the pet look broken in ways that don't point at +themselves — missing server config (the controller exits its thread at startup, +so the pet appears alive and simply never answers), no input device, no OCR +engine behind `petctl read`, a wake model that isn't where `.env` says, a +silence timeout long enough to feel like lag. Each line says what to do about +it, not just what's wrong. It doesn't touch the network or open the microphone +unless you add `--deep`, so it's safe to run when the network is the suspect. + +## Little things + +The pet **remembers where you left it** — drag it somewhere deliberate and +that's where it starts next time. If that position is on a monitor you've since +unplugged it goes back to the default corner rather than restoring itself +somewhere off-screen. `PET_REMEMBER_POSITION=false` to always start in the +corner. + +Its **mouth moves with the actual audio** rather than flapping on a timer: the +PCM going to the speakers is reduced to a loudness per frame and that picks the +talking sprite, so the pet shuts up when the voice pauses. Offline `pyttsx3` +playback has no waveform to follow, so it falls back to the timed loop. + ## Project layout ``` @@ -148,18 +175,23 @@ bolt_pet/ state.py PetState enum + a small transition-checked state machine server_client.py /desk/converse, /desk/tool_result, /desk/report_status controller.py the pipeline: wake word -> STT -> server -> TTS, on a QThread - speech_text.py strips markdown/emoji/URLs so the voice never says "asterisk" + speech.py what the pet says and how each kind of saying behaves + speech_text.py strips markdown/emoji/URLs so the voice never says "asterisk" pet_actions.py petctl move/emote/say/wander/nap parsing screen_context.py active-window title + fullscreen detection quiet.py quiet-hours schedule notifications.py desktop notification bridge (Linux/D-Bus) history.py rolling conversation transcript hotkey.py global push-to-talk (pynput, optional) + window_state.py remembers where you left the pet + doctor.py `--doctor` preflight: is this install going to work? audio/ mic.py input stream + energy-based VAD utterance capture wake_word.py openWakeWord thunderbolt.onnx detection (see above) - stt.py Deepgram - tts.py ElevenLabs streaming PCM, offline pyttsx3 fallback + stt.py Deepgram (one-shot) + stt_stream.py Deepgram live websocket — transcribes while you speak + tts.py ElevenLabs streaming PCM, offline pyttsx3 fallback, + plus the loudness envelope that drives the mouth barge_in.py "you started talking" detector, to cut playback short ui/ app.py wires QApplication + window + tray + controller thread together @@ -172,8 +204,10 @@ bolt_pet/ scripts/ slice_spritesheet.py cuts a grid sprite sheet into the per-frame convention tests/ pure-logic unit tests (state machine, wake-phrase - matching, HTTP client against mocks) — nothing here - needs real audio hardware or a display + matching, HTTP client against mocks) plus one + end-to-end smoke test that drives whole turns against + a real local HTTP server — nothing here needs real + audio hardware or a display ``` ## Security notes @@ -197,7 +231,6 @@ notifications. No screenshots or images are ever sent. ## Known limitations / not-yet-done -- Pet screen position isn't persisted across restarts. - Wandering is a straight walk to a random point — no Shimeji-style physics, wall-climbing or falling. - Push-to-talk and the notification bridge are platform-limited: the hotkey diff --git a/bolt_pet/__main__.py b/bolt_pet/__main__.py index 22daeb1..868f596 100644 --- a/bolt_pet/__main__.py +++ b/bolt_pet/__main__.py @@ -1,8 +1,15 @@ -"""Entry point: python -m bolt_pet""" +"""Entry point: python -m bolt_pet [--doctor [--deep]]""" import sys -from .ui.app import run - if __name__ == "__main__": + if "--doctor" in sys.argv: + # Imported inside the branch, not at module scope: the doctor exists to + # diagnose installs where importing the UI would itself blow up. + from .doctor import main as doctor + + sys.exit(doctor(sys.argv[1:])) + + from .ui.app import run + sys.exit(run()) diff --git a/bolt_pet/assets/sprites/idle/frame_00.png b/bolt_pet/assets/sprites/idle/frame_00.png index 3b01320..fecd36b 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_00.png and b/bolt_pet/assets/sprites/idle/frame_00.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_01.png b/bolt_pet/assets/sprites/idle/frame_01.png index 0d1cc60..69fe990 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_01.png and b/bolt_pet/assets/sprites/idle/frame_01.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_02.png b/bolt_pet/assets/sprites/idle/frame_02.png index f7a055c..8abfcf9 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_02.png and b/bolt_pet/assets/sprites/idle/frame_02.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_03.png b/bolt_pet/assets/sprites/idle/frame_03.png index da58383..6d4fd50 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_03.png and b/bolt_pet/assets/sprites/idle/frame_03.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_04.png b/bolt_pet/assets/sprites/idle/frame_04.png index 3b01320..92f4ff5 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_04.png and b/bolt_pet/assets/sprites/idle/frame_04.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_05.png b/bolt_pet/assets/sprites/idle/frame_05.png index 55c6a70..700d555 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_05.png and b/bolt_pet/assets/sprites/idle/frame_05.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_06.png b/bolt_pet/assets/sprites/idle/frame_06.png index e21d5d2..66b151c 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_06.png and b/bolt_pet/assets/sprites/idle/frame_06.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_07.png b/bolt_pet/assets/sprites/idle/frame_07.png index ba3c27c..ad5cd07 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_07.png and b/bolt_pet/assets/sprites/idle/frame_07.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_00.png b/bolt_pet/assets/sprites/listening/frame_00.png index d4872c5..3f10756 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_00.png and b/bolt_pet/assets/sprites/listening/frame_00.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_01.png b/bolt_pet/assets/sprites/listening/frame_01.png index 93f44ee..88a1ab2 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_01.png and b/bolt_pet/assets/sprites/listening/frame_01.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_02.png b/bolt_pet/assets/sprites/listening/frame_02.png index ee23890..39b59fd 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_02.png and b/bolt_pet/assets/sprites/listening/frame_02.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_03.png b/bolt_pet/assets/sprites/listening/frame_03.png index c2bbf76..5f07e44 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_03.png and b/bolt_pet/assets/sprites/listening/frame_03.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_04.png b/bolt_pet/assets/sprites/listening/frame_04.png new file mode 100644 index 0000000..729e550 Binary files /dev/null and b/bolt_pet/assets/sprites/listening/frame_04.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_05.png b/bolt_pet/assets/sprites/listening/frame_05.png new file mode 100644 index 0000000..b5ed0f0 Binary files /dev/null and b/bolt_pet/assets/sprites/listening/frame_05.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_00.png b/bolt_pet/assets/sprites/talking/frame_00.png index 8559181..0de4378 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_00.png and b/bolt_pet/assets/sprites/talking/frame_00.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_01.png b/bolt_pet/assets/sprites/talking/frame_01.png index 818e23c..2953b89 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_01.png and b/bolt_pet/assets/sprites/talking/frame_01.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_02.png b/bolt_pet/assets/sprites/talking/frame_02.png index 2974be7..47f2497 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_02.png and b/bolt_pet/assets/sprites/talking/frame_02.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_03.png b/bolt_pet/assets/sprites/talking/frame_03.png index 8c930e4..6e1930d 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_03.png and b/bolt_pet/assets/sprites/talking/frame_03.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_04.png b/bolt_pet/assets/sprites/talking/frame_04.png new file mode 100644 index 0000000..357e401 Binary files /dev/null and b/bolt_pet/assets/sprites/talking/frame_04.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_05.png b/bolt_pet/assets/sprites/talking/frame_05.png new file mode 100644 index 0000000..6c4eae5 Binary files /dev/null and b/bolt_pet/assets/sprites/talking/frame_05.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_00.png b/bolt_pet/assets/sprites/thinking/frame_00.png index cf5da13..3499dad 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_00.png and b/bolt_pet/assets/sprites/thinking/frame_00.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_01.png b/bolt_pet/assets/sprites/thinking/frame_01.png index a033ebe..e59ffef 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_01.png and b/bolt_pet/assets/sprites/thinking/frame_01.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_02.png b/bolt_pet/assets/sprites/thinking/frame_02.png index 5da2a87..c0a8f81 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_02.png and b/bolt_pet/assets/sprites/thinking/frame_02.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_03.png b/bolt_pet/assets/sprites/thinking/frame_03.png index 986f326..74a8967 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_03.png and b/bolt_pet/assets/sprites/thinking/frame_03.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_04.png b/bolt_pet/assets/sprites/thinking/frame_04.png index 0729e2e..c694357 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_04.png and b/bolt_pet/assets/sprites/thinking/frame_04.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_05.png b/bolt_pet/assets/sprites/thinking/frame_05.png index 0729e2e..f74e595 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_05.png and b/bolt_pet/assets/sprites/thinking/frame_05.png differ diff --git a/bolt_pet/audio/mic.py b/bolt_pet/audio/mic.py index 52beaaa..e3a6a3c 100644 --- a/bolt_pet/audio/mic.py +++ b/bolt_pet/audio/mic.py @@ -37,6 +37,20 @@ def rms(frame: np.ndarray) -> float: return float(np.sqrt(np.mean(frame.astype(np.float64) ** 2))) +def _emit(on_frame, frame) -> None: + """Hand a frame to a listener without letting it break the capture. + + Streaming STT is an optimisation riding along with recording; if its + socket dies mid-utterance the recording must carry on untouched, because + the one-shot fallback is about to need the full buffer.""" + if on_frame is None: + return + try: + on_frame(frame) + except Exception: + pass + + def record_utterance( stream: AudioStream, should_continue=lambda: True, @@ -47,6 +61,7 @@ def record_utterance( grace_s: float = None, frame_len: int = config.FRAME_LEN, sample_rate: int = config.SAMPLE_RATE, + on_frame=None, ) -> Optional[np.ndarray]: """Capture one utterance from *stream*: wait for speech to start, stop after trailing silence. Returns None if nothing usable was heard. @@ -55,6 +70,11 @@ def record_utterance( (e.g. the pet window was closed) without needing threading primitives baked into this function. + *on_frame*, if given, is called with each captured frame while speech is + in progress — that is how streaming STT transcribes as you talk rather + than after (see audio/stt_stream.py). It is fire-and-forget: this function + still returns the full buffer, so a failed stream costs nothing. + *grace_s* is how long to wait for speech to *begin* before giving up. The controller stretches it for follow-up questions, where you're being asked something and need a moment to think rather than having just said @@ -83,10 +103,12 @@ def record_utterance( if frame_rms >= rms_threshold: started = True frames.append(frame) + _emit(on_frame, frame) elif waited > grace_frames: return None # woke it up but said nothing continue frames.append(frame) + _emit(on_frame, frame) if frame_rms < rms_threshold: silence_frames += 1 if silence_frames >= silence_limit: diff --git a/bolt_pet/audio/stt_stream.py b/bolt_pet/audio/stt_stream.py new file mode 100644 index 0000000..371c9fe --- /dev/null +++ b/bolt_pet/audio/stt_stream.py @@ -0,0 +1,181 @@ +"""Streaming speech-to-text — transcribing *while* you talk, not after. + +The one-shot path (`stt.transcribe`) waits for the utterance to finish, then +uploads the whole WAV and waits again. That second wait is dead time between +you stopping and the pet reacting, and it grows with the length of what you +said — a thirty-second question costs noticeably more than a five-second one. + +Deepgram's live endpoint removes it: frames go up as they are captured, so by +the time the VAD decides you have stopped, the transcript is essentially +already there. Same model, same account, same accuracy — the difference is +purely when the work happens. + +Design constraints that shaped this: + +- **Failure must be invisible.** No websocket, no network, a mid-utterance + disconnect — all of it falls back to the one-shot path, which still has the + full audio buffered. Streaming is an optimisation, never a dependency, so + `open()` returning None is an ordinary outcome rather than an error. +- **The VAD still decides when you stopped.** Deepgram has its own endpointing + and using it would save more, but it would also move a decision the rest of + the pipeline is built around (barge-in, follow-up listening, the grace + period) into a remote service. Not worth coupling those to the network on + the first pass. +- **The socket is per-utterance.** Holding one open across an idle pet would + bill for silence and drop on the first network blip; opening one takes + ~100ms, which is already inside the time it takes a person to start talking. + +The websocket client is injectable, so the whole protocol — send frames, read +`is_final` transcripts, close, take the result — is tested without a network. +""" + +from __future__ import annotations + +import json +import logging +import threading +from typing import Callable, Optional + +import numpy as np + +from .. import config + +logger = logging.getLogger("bolt_pet.stt_stream") + +_ENDPOINT = ( + "wss://api.deepgram.com/v1/listen" + "?encoding=linear16&channels=1&sample_rate={rate}&model={model}" + "&language=en&smart_format=true&interim_results=false" +) + + +def available() -> bool: + """Whether streaming STT can even be attempted in this install.""" + if not config.STT_STREAMING or not config.DEEPGRAM_API_KEY: + return False + try: + import websocket # noqa: F401 (websocket-client) + return True + except Exception: + return False + + +class StreamingTranscriber: + """One utterance's worth of live transcription. + + Usage mirrors how the capture loop already works — feed frames as they + arrive, then ask what was said: + + session = StreamingTranscriber.open() + ... + session.feed(frame) # per mic frame, non-blocking + text = session.finish() # after the VAD says you stopped + """ + + def __init__(self, socket, *, sample_rate: int = None): + self._socket = socket + self._sample_rate = sample_rate or config.SAMPLE_RATE + self._transcript: list[str] = [] + self._lock = threading.Lock() + self._closed = False + self._reader = threading.Thread( + target=self._read_loop, name="stt-stream-reader", daemon=True) + self._reader.start() + + # -- lifecycle ---------------------------------------------------------- + + @classmethod + def open(cls, *, connect: Optional[Callable] = None, + sample_rate: int = None) -> Optional["StreamingTranscriber"]: + """Connect, or return None if streaming isn't possible right now. + + None is a normal outcome, not a failure: the caller keeps the audio and + falls back to the one-shot upload.""" + if connect is None and not available(): + return None + rate = sample_rate or config.SAMPLE_RATE + try: + if connect is not None: + socket = connect() + else: + import websocket + + socket = websocket.create_connection( + _ENDPOINT.format(rate=rate, model=config.DEEPGRAM_MODEL), + header={"Authorization": f"Token {config.DEEPGRAM_API_KEY}"}, + timeout=10, + ) + return cls(socket, sample_rate=rate) + except Exception as exc: + logger.info("Streaming STT unavailable (%s) — using the one-shot path.", exc) + return None + + def feed(self, frame: np.ndarray) -> None: + """Send one captured frame. Never raises — a dead socket just means the + fallback will do the work.""" + if self._closed: + return + try: + self._socket.send_binary(np.asarray(frame, dtype=np.int16).tobytes()) + except Exception: + logger.debug("Streaming STT send failed; abandoning the stream", exc_info=True) + self._closed = True + + def finish(self, timeout: float = 3.0) -> str: + """Close the stream and return whatever was transcribed. + + Deepgram flushes its final results after the close frame, so this waits + briefly for the reader — bounded, because a hung socket must not hold + up the reply.""" + if not self._closed: + try: + self._socket.send(json.dumps({"type": "CloseStream"})) + except Exception: + pass + self._closed = True + self._reader.join(timeout=timeout) + try: + self._socket.close() + except Exception: + pass + with self._lock: + return " ".join(part for part in self._transcript if part).strip() + + # -- the reader --------------------------------------------------------- + + def _read_loop(self) -> None: + while not self._closed: + try: + message = self._socket.recv() + except Exception: + break + if not message: + break + text, is_final = self._parse(message) + if text and is_final: + with self._lock: + self._transcript.append(text) + + @staticmethod + def _parse(message) -> tuple[str, bool]: + """Pull (text, is_final) out of a Deepgram results frame. + + Tolerant on purpose: anything unrecognised is ignored rather than + raising on the reader thread, where an exception would silently kill + transcription for the rest of the utterance.""" + try: + if isinstance(message, bytes): + message = message.decode("utf-8", "ignore") + data = json.loads(message) + except (TypeError, ValueError): + return "", False + if not isinstance(data, dict): + return "", False + alternatives = ( + ((data.get("channel") or {}).get("alternatives") or []) + if data.get("type") in (None, "Results") else [] + ) + if not alternatives: + return "", False + text = str((alternatives[0] or {}).get("transcript") or "").strip() + return text, bool(data.get("is_final") or data.get("speech_final")) diff --git a/bolt_pet/audio/tts.py b/bolt_pet/audio/tts.py index a03cb89..264902f 100644 --- a/bolt_pet/audio/tts.py +++ b/bolt_pet/audio/tts.py @@ -14,6 +14,7 @@ fallback has no such concept and always sounds like itself. from __future__ import annotations +import time from typing import Iterable, Iterator, Optional import numpy as np @@ -162,32 +163,81 @@ def synthesize_dialogue( return pcm, config.TTS_SAMPLE_RATE -def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool: +# ── how loud is it right now ──────────────────────────────────────────────── +# The PCM is already decoded here on its way to the speakers, so the amplitude +# envelope is free — and it is exactly what a mouth needs to move in time with +# speech. Throwing it away and animating the mouth on a timer instead is why +# most talking sprites look dubbed. + +# int16 RMS that counts as "mouth fully open". Speech peaks around 8-12k; +# 6000 keeps normal talking in the upper half of the range without clipping +# every syllable to wide-open. +_LOUD_RMS = 6000.0 + + +def level_of(frame: np.ndarray) -> float: + """0..1 loudness for one chunk of PCM. + + Square-rooted because perceived loudness is not linear in amplitude — a + linear mapping leaves the mouth barely moving through ordinary speech.""" + if frame is None or len(frame) == 0: + return 0.0 + rms = float(np.sqrt(np.mean(np.square(frame.astype(np.float32))))) + return float(min(1.0, (rms / _LOUD_RMS) ** 0.5)) + + +def envelope(pcm: np.ndarray, sample_rate: int, fps: int = 30) -> list: + """Per-frame loudness for a whole clip, for playback that isn't streamed.""" + if pcm is None or len(pcm) == 0: + return [] + window = max(1, int(sample_rate / max(1, fps))) + return [level_of(pcm[start:start + window]) for start in range(0, len(pcm), window)] + + +def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None, + on_level=None) -> bool: """Play a whole clip. Returns True if it finished, False if *should_stop* (barge-in) cut it short. *should_stop* is polled while audio plays — each poll consumes one mic frame, which is what paces this loop.""" import sounddevice as sd + levels = envelope(pcm, sample_rate) if on_level is not None else [] + started = time.monotonic() sd.play(pcm, samplerate=sample_rate, device=config.SPEAKER_DEVICE) if not blocking: return True - if should_stop is None: + if should_stop is None and on_level is None: sd.wait() return True while True: + if levels: + # Indexed by elapsed time rather than by chunk, because this path + # hands the whole clip to the device at once and never sees it + # again — wall clock is the only position we have. + index = int((time.monotonic() - started) * 30) + if index < len(levels): + try: + on_level(levels[index]) + except Exception: + levels = [] try: if not sd.get_stream().active: break except Exception: break # stream already torn down — playback is over - if should_stop(): + if should_stop is not None and should_stop(): sd.stop() return False return True -def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None) -> bool: - """Play int16 chunks as they arrive. Returns False if interrupted.""" +def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None, + on_level=None) -> bool: + """Play int16 chunks as they arrive. Returns False if interrupted. + + *on_level* receives each chunk's loudness (0..1) just before it is written, + which is what drives the mouth: the sprite is animated by the same audio + the speakers are getting, not by a guess about how long a word takes.""" import sounddevice as sd with sd.OutputStream( @@ -199,6 +249,11 @@ def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None # now, not at the end of the buffered chunk. out.abort() return False + if on_level is not None: + try: + on_level(level_of(chunk)) + except Exception: + on_level = None # a broken listener must not stop playback out.write(chunk) return True @@ -213,7 +268,8 @@ def speak_offline(text: str) -> None: engine.runAndWait() -def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None) -> bool: +def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None, + on_level=None) -> bool: """Speak *text*, preferring streaming ElevenLabs, then whole-clip ElevenLabs, then offline TTS. *on_error*, if given, is called with the exception when ElevenLabs fails (useful for logging) — a fallback still @@ -233,13 +289,14 @@ def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = stream_pcm(text, voice_id=voice_id), config.TTS_SAMPLE_RATE, should_stop=should_stop, + on_level=on_level, ) except TtsError as exc: if on_error is not None: on_error(exc) try: pcm, sample_rate = synthesize_pcm(text, voice_id=voice_id) - return play_pcm(pcm, sample_rate, should_stop=should_stop) + return play_pcm(pcm, sample_rate, should_stop=should_stop, on_level=on_level) except TtsError as exc: if on_error is not None: on_error(exc) diff --git a/bolt_pet/config.py b/bolt_pet/config.py index ca1d62e..2950e71 100644 --- a/bolt_pet/config.py +++ b/bolt_pet/config.py @@ -58,6 +58,11 @@ WAKE_CHECK_INTERVAL_SECONDS = float(os.environ.get("WAKE_CHECK_INTERVAL_SECONDS" DEEPGRAM_API_KEY = os.environ.get("DEEPGRAM_API_KEY", "") DEEPGRAM_MODEL = os.environ.get("DEEPGRAM_MODEL", "nova-3") +# Transcribe *while* you talk instead of uploading the finished clip: frames go +# up as they are captured, so the transcript is ready the moment the VAD says +# you stopped. Needs `websocket-client`; falls back to the one-shot upload +# whenever it can't connect, so turning it on can only help. +STT_STREAMING = os.environ.get("STT_STREAMING", "true").lower() in ("1", "true", "yes", "on") # ── TTS (ElevenLabs, requested as raw PCM so playback needs no external # player binary — cross-platform via sounddevice instead of shelling out to @@ -168,6 +173,13 @@ BARGE_IN_FRAMES = int(os.environ.get("BARGE_IN_FRAMES", "4")) # consecutive lou # waking the pet from idle (useful if Bolt's own voice trips the model). BARGE_IN_WAKE_THRESHOLD = float(os.environ.get("BARGE_IN_WAKE_THRESHOLD") or 0) or None +# ── streaming the reply ───────────────────────────────────────────────────── +# Speak each sentence as the server produces it, instead of waiting out the +# whole model call before the first word. Falls back to the ordinary +# request/response path automatically if the server has no streaming endpoint +# or the stream fails before anything has been spoken. +STREAMING_REPLIES = os.environ.get("STREAMING_REPLIES", "true").lower() in ("1", "true", "yes", "on") + # ── streaming TTS ─────────────────────────────────────────────────────────── # ElevenLabs' /stream endpoint + chunked playback: the pet starts talking # after the first PCM chunk instead of after the whole clip is synthesized. @@ -307,6 +319,10 @@ PET_WANDER_MARGIN = int(os.environ.get("PET_WANDER_MARGIN", "20")) # keep off s PET_SHAPED_INPUT = os.environ.get("PET_SHAPED_INPUT", "true").lower() in ("1", "true", "yes", "on") PET_CLICK_THROUGH = os.environ.get("PET_CLICK_THROUGH", "false").lower() in ("1", "true", "yes", "on") # Snap flush to a screen edge when dropped/parked within this many pixels of it. +# Start where it was left rather than in the bottom-right corner. Ignored when +# PET_START_X/Y pin it explicitly, and a saved position on a monitor that is no +# longer plugged in is discarded rather than hiding the pet offscreen. +PET_REMEMBER_POSITION = os.environ.get("PET_REMEMBER_POSITION", "true").lower() in ("1", "true", "yes", "on") PET_EDGE_SNAP = os.environ.get("PET_EDGE_SNAP", "true").lower() in ("1", "true", "yes", "on") PET_SNAP_MARGIN = int(os.environ.get("PET_SNAP_MARGIN", "48")) diff --git a/bolt_pet/controller.py b/bolt_pet/controller.py index 9a840d3..4938e0b 100644 --- a/bolt_pet/controller.py +++ b/bolt_pet/controller.py @@ -20,15 +20,20 @@ from typing import Optional from PySide6.QtCore import QObject, Signal from . import ( - config, dialogue as dialogue_mod, file_delivery, file_ops, + config, dialogue as dialogue_mod, speech, file_delivery, file_ops, history as history_mod, monitors as monitors_mod, notifications, pet_actions, quiet, screen_context, screen_text, self_restart, server_client, speech_text, updater, ) from . import __version__ -from .audio import barge_in, mic, stt, tts, wake_word +from .audio import barge_in, mic, stt, stt_stream, tts, wake_word from .state import PetState, PetStateMachine +# A burst while the pet is busy is batched into one turn, but the queue still +# needs a ceiling — a notification storm must not become an unbounded backlog +# that gets read out minutes later. +_MAX_PENDING_NOTIFICATIONS = 12 + # How often to re-check whether the pet should be napping. The fullscreen # probe shells out to xprop, so this deliberately isn't every heartbeat tick. _NAP_CHECK_INTERVAL_SECONDS = 10.0 @@ -41,6 +46,7 @@ class PetController(QObject): action = Signal(dict) # parsed petctl action for the UI to perform napping = Signal(bool) # quiet hours / fullscreen do-not-disturb voice_changed = Signal(str) # name of the server-picked voice ("" = default) + mouth = Signal(float) # 0..1 speech loudness, for lip-sync while talking restart_requested = Signal(str) # version we just updated to finished = Signal() @@ -77,6 +83,17 @@ class PetController(QObject): self._voice_name = "" self._barge_in: Optional[barge_in.BargeInDetector] = None + # Everything the pet says goes through here; see speech.py for why the + # four hand-rolled copies of this became one. + self._speaker = speech.Speaker( + state=self._state, tts=tts, + history=lambda text: self.history.add(history_mod.PET, text, time.time()), + on_said=self.said.emit, on_log=self.log.emit, + barge_in=lambda: self._barge_in, + detail_of=self._barge_in_detail, + voice_id=lambda: self._voice_id, + on_level=self.mouth.emit, + ) self._napping = False self._nap_forced: Optional[bool] = None # petctl nap on/off overrides the schedule self._last_nap_check = 0.0 @@ -249,21 +266,28 @@ class PetController(QObject): self._follow_ups = 0 self._state.transition(PetState.LISTENING) + # Transcribe while they talk rather than after: frames go to Deepgram + # as they are captured, so the text is ready the moment the VAD says + # they stopped. None here just means the one-shot path will do it. + streamed = stt_stream.StreamingTranscriber.open() pcm = mic.record_utterance( self._stream, should_continue=self._should_continue, # Answering a question deserves longer than saying the wake word # on purpose does — you were just asked something. grace_s=config.FOLLOW_UP_GRACE_SECONDS if following_up else None, + on_frame=streamed.feed if streamed is not None else None, ) if pcm is None: + if streamed is not None: + streamed.finish() self._follow_ups = 0 # silence ends the chain self._state.transition(PetState.IDLE) return self._state.transition(PetState.THINKING) try: - text = stt.transcribe(pcm) + text = self._transcribe(pcm, streamed) except stt.SttError as exc: self.log.emit(f"STT failed: {exc}") self._state.transition(PetState.ERROR) @@ -278,9 +302,7 @@ class PetController(QObject): try: # What's focused right now rides along, so "what's this error?" # has a referent without you having to describe the window. - reply = server_client.converse( - self._with_context(text), on_command=self._handle_command - ) + reply = self._ask_server(self._with_context(text)) except server_client.ServerError as exc: self.log.emit(f"Server error: {exc}") self._state.transition(PetState.ERROR) @@ -289,10 +311,34 @@ class PetController(QObject): self._check_deliveries() self._apply_voice(reply) - self._speak(reply.text) + if reply.spoken: + # Streamed: every sentence was spoken and logged as it arrived. + # The text still matters — a reply ending on a question should keep + # the mic open — but saying it again would repeat the whole answer. + self._after_speaking(reply.text, completed=True) + else: + self._speak(reply.text) self._state.transition(PetState.IDLE) self._maybe_self_restart() + def _transcribe(self, pcm, streamed) -> str: + """The transcript, from the live stream if it produced one. + + The fallback is not a rare path to be tolerated — it is the safety net + that lets streaming be switched on at all. Whatever happened to the + socket, the full audio is still buffered here, so a failed stream costs + one ordinary upload and nothing else.""" + if streamed is not None: + try: + text = streamed.finish() + except Exception: + self.log.emit("Streaming STT failed — falling back.") + text = "" + if text: + self.log.emit("(transcribed while you spoke)") + return text + return stt.transcribe(pcm) + def _with_context(self, text: str) -> str: """Everything the server gets alongside what you actually said: the focused window title, and a one-line note about the screen layout so @@ -492,6 +538,26 @@ class PetController(QObject): self._speak(reply.text) self._state.force(PetState.IDLE) + def _ask_server(self, text: str): + """One turn with the server, streamed when possible. + + Streaming speaks each sentence as it is generated, so the wait is + time-to-first-sentence rather than the whole model call. It falls back + to the ordinary request/response path when the server has no streaming + endpoint, or when a stream dies *before* anything was spoken — after + that, retrying would say the first half twice.""" + if config.STREAMING_REPLIES: + try: + return server_client.converse_stream( + text, on_say=self._speak_stream_chunk, + on_command=self._handle_command, + ) + except server_client.ServerError as exc: + self.log.emit(f"Streaming unavailable ({exc}) — using the plain path.") + return server_client.converse( + text, on_command=self._handle_command, on_say=self._speak_holding, + ) + def _play_dialogue(self, scene: dict) -> str: """Play a `dialoguectl` scene and report back up the relay. @@ -525,20 +591,8 @@ class PetController(QObject): # scene or fix the voice, and try again inside the same turn. return f"[dialogue] couldn't synthesize it: {exc}" - resume = self._state.state - self._state.transition(PetState.TALKING) - self.said.emit(speech_text.for_display(text)) - self.history.add(history_mod.PET, text, time.time()) - - should_stop = None - if self._barge_in is not None: - self._barge_in.reset() - should_stop = self._barge_in.check - completed = tts.play_pcm(pcm, sample_rate, should_stop=should_stop) - if self._barge_in is not None: - self._barge_in.reset() # the pet's own voices are in the wake window - if resume in (PetState.THINKING, PetState.IDLE): - self._state.transition(resume) + completed = self._speaker.say_pcm( + speech.Utterance.scene(text), pcm=pcm, sample_rate=sample_rate) if not completed: return dialogue_mod.describe(scene) + " (interrupted — they talked over it)" @@ -569,32 +623,30 @@ class PetController(QObject): self.reset_voice() def _speak(self, text: str) -> None: - self._state.transition(PetState.TALKING) - # Bubble gets the markdown stripped but emoji kept (it can't render - # **bold** but draws emoji fine); tts.speak() does its own, stricter - # sanitizing for the voice. - self.said.emit(speech_text.for_display(text)) - self.log.emit(f"Bolt: {text}") - self.history.add(history_mod.PET, text, time.time()) + """The answer: transcript, bubble, and the follow-up rule.""" + completed = self._speaker.say(speech.Utterance.reply(text)) + self._after_speaking(text, completed=completed, + detail=self._speaker.last_detail) - should_stop = None - if self._barge_in is not None: - self._barge_in.reset() - should_stop = self._barge_in.check - completed = tts.speak( - text, - on_error=lambda exc: self.log.emit(f"TTS failed: {exc}"), - should_stop=should_stop, - voice_id=self._voice_id or None, - ) - # Read the scoring history *before* resetting, or the log reports the - # blank counters instead of what actually fired. - detail = self._barge_in_detail() - if self._barge_in is not None: - # Playback fed the pet's own voice into the wake model's rolling - # window. Clear it before the idle listener starts scoring again, - # or Bolt's last sentence is still in there being re-scored. - self._barge_in.reset() + def _speak_holding(self, text: str) -> None: + """"Give me a sec" while a tool runs — filler, so no transcript, and it + resumes the state it interrupted because the turn isn't over.""" + self._speaker.say(speech.Utterance.holding(text)) + + def _speak_stream_chunk(self, text: str) -> None: + """One sentence of a streamed answer, spoken the moment it arrives.""" + completed = self._speaker.say(speech.Utterance.stream_chunk(text)) + if not completed: + self.log.emit("Interrupted — listening.") + self._follow_ups = 0 + self._talk_now.set() + + def _after_speaking(self, text: str, *, completed: bool, detail: str = "") -> None: + """What happens once an answer has been said, however it was said. + + Shared by the plain and streamed paths: a streamed reply is spoken + sentence by sentence, but it still has to obey the same rules about + keeping the mic open when it ended on a question.""" if not completed: # You talked over it — take that as the start of the next turn # rather than making you say the wake word again. @@ -614,20 +666,22 @@ class PetController(QObject): def _should_follow_up(self, text: str) -> bool: """Whether *text* leaves the pet waiting on an answer. - Muted is excluded because mute means "don't listen to me" — an - automatic turn would walk straight past it. Napping isn't: quiet - hours suppress the pet *starting* something, and a question is only - ever asked in reply to you.""" - if not config.FOLLOW_UP_LISTEN or self._muted: + The rule itself — question, cap, off switch — is + `speech.follow_up_decision`, because it is a rule with an off-by-one in + it and deserves a test that needs no audio. What stays here is the part + that is genuinely the controller's: mute, and the log line. Muted is + excluded because mute means "don't listen to me" and an automatic turn + would walk straight past it. Napping isn't: quiet hours suppress the + pet *starting* something, and a question is only ever asked in reply to + you.""" + if self._muted: return False - if not speech_text.is_question(text): - return False - # Only worth mentioning the cap on a reply that would otherwise have - # kept listening, or it fires on every statement the pet makes. - if config.FOLLOW_UP_MAX_TURNS > 0 and self._follow_ups >= config.FOLLOW_UP_MAX_TURNS: + keep, why = speech.follow_up_decision(text, completed=True, follow_ups=self._follow_ups) + if not keep and "cap" in why: + # Only worth mentioning the cap on a reply that would otherwise have + # kept listening, or it fires on every statement the pet makes. self.log.emit("Follow-up limit reached — say the wake word to keep going.") - return False - return True + return keep def _barge_in_detail(self) -> str: """Why the interruption fired, for the log. How far into playback it @@ -685,33 +739,54 @@ class PetController(QObject): def _queue_notification(self, notification: notifications.Notification) -> None: """Called on the watcher thread — just queue it; forwarding happens on - the pipeline thread where it can't collide with a live conversation.""" - if not self._notification_gate.should_forward(notification, time.monotonic()): + the pipeline thread where it can't collide with a live conversation. + + Only the *filter* applies here. The rate limit used to as well, which + meant a second message arriving inside the window was silently thrown + away — with NOTIFICATION_MIN_INTERVAL_SECONDS=60, two texts a minute + apart and you only ever heard about one of them. Losing a message from + a person to save a round trip is the wrong trade; they are batched at + the far end instead, which costs the same one round trip and keeps + them all.""" + if not self._notification_gate.matches(notification): return with self._notification_lock: + if len(self._pending_notifications) >= _MAX_PENDING_NOTIFICATIONS: + self._pending_notifications.pop(0) # bound it; oldest goes first self._pending_notifications.append(notification) def _drain_notifications(self) -> None: + """Forward everything waiting as ONE turn. + + Batching is what makes it safe to keep every notification: five that + arrived while the pet was mid-conversation become one message and one + round trip, instead of five separate interruptions queued up to fire + back to back.""" with self._notification_lock: pending, self._pending_notifications = self._pending_notifications, [] + if not pending or not self._running or self._napping: + return for notification in pending: - if not self._running or self._napping: - return self.log.emit(f"Notification: {notification.as_text()}") self.history.add(history_mod.SYSTEM, notification.as_text(), time.time()) - try: - reply = server_client.converse( - f"[desktop notification] {notification.as_text()}", - on_command=self._handle_command, - ) - except server_client.ServerError as exc: - self.log.emit(f"Couldn't forward notification: {exc}") - return - self._check_deliveries() - self._apply_voice(reply) - if reply.text.strip(): - self._speak(reply.text) - self._state.transition(PetState.IDLE) + + if len(pending) == 1: + message = f"[desktop notification] {pending[0].as_text()}" + else: + lines = "\n".join(f"- {n.as_text()}" for n in pending) + message = f"[{len(pending)} desktop notifications]\n{lines}" + try: + reply = server_client.converse( + message, on_command=self._handle_command, on_say=self._speak_holding, + ) + except server_client.ServerError as exc: + self.log.emit(f"Couldn't forward notification: {exc}") + return + self._check_deliveries() + self._apply_voice(reply) + if reply.text.strip() and not reply.spoken: + self._speak(reply.text) + self._state.transition(PetState.IDLE) # ── file delivery ──────────────────────────────────────────────────── @@ -784,20 +859,36 @@ class PetController(QObject): # ── heartbeat ──────────────────────────────────────────────────────── def _maybe_heartbeat(self) -> None: + """Called from the wake listener's tick (~every WAKE_CHECK_INTERVAL_SECONDS). + + Two different cadences live here, and conflating them was costing + minutes. A *notification* is an event that already happened — it should + go out as soon as the pet is free, which is the next tick. The + *heartbeat* is a poll, and polling the server every 1.2s would be + absurd, so it stays on its own interval.""" self._refresh_nap_state() self._maybe_update() if self._update_pending: return # on the way out — don't start a conversation now + + # Notifications: every tick, not every heartbeat. + if self._state.state == PetState.IDLE and not self._napping: + self._drain_notifications() + now = time.monotonic() if now - self._last_heartbeat < config.HEARTBEAT_INTERVAL_SECONDS: return - self._last_heartbeat = now if self._state.state != PetState.IDLE: + # Mid-conversation. Do NOT stamp the clock — an earlier version + # did, so a heartbeat that landed while the pet was talking burned + # its slot and waited another full interval. With follow-up + # listening that could repeat for several cycles, which is why a + # notification could sit unspoken for five or ten minutes. return if self._napping: return # quiet hours: still answers when spoken to, just doesn't start + self._last_heartbeat = now self._check_deliveries() - self._drain_notifications() if self._state.state != PetState.IDLE: return try: diff --git a/bolt_pet/dialogue.py b/bolt_pet/dialogue.py index 7665031..b95028f 100644 --- a/bolt_pet/dialogue.py +++ b/bolt_pet/dialogue.py @@ -224,4 +224,12 @@ def describe(action: dict, *, played: bool = True) -> str: return ( f"[dialogue] played {len(lines)} line{'s' if len(lines) != 1 else ''} " f"in {len(voices)} voice{'s' if len(voices) != 1 else ''}: {', '.join(voices)}{note}" + # The scene was spoken out loud before this result got back to the + # server, and the model has no other way to know that. Without saying + # so it writes a final reply summarising what the user just heard, and + # the pet says the same thing twice in a row (observed 2026-07-31). + # An instruction delivered here, at the moment it applies, lands far + # better than a rule buried in a long system prompt. + "\nThe user HEARD this already. Do not repeat, summarise or narrate it " + "in your reply — answer with at most one short line, or nothing new." ) diff --git a/bolt_pet/doctor.py b/bolt_pet/doctor.py new file mode 100644 index 0000000..7f58c85 --- /dev/null +++ b/bolt_pet/doctor.py @@ -0,0 +1,272 @@ +"""`python -m bolt_pet --doctor` — is this install actually going to work? + +Written after a week of debugging things a preflight would have shown in one +command: a missing port publish, a wrong reverse-proxy header, a venv whose +python was a zero-byte file, an OCR engine that was never installed. Every one +of those presented as "the pet is being weird" and took a conversation to find. + +Each check is independent and reports one of three things — ok, a warning +(works, but degraded), or a failure (this will not do what you expect) — and +says *what to do about it* rather than just what is wrong. Nothing here raises: +a doctor that crashes on a broken install is diagnosing the wrong patient. + +Deliberately does not open the mic or call a paid API by default. It checks +that the door is unlocked, not that the room is furnished; `--deep` is there +when you want it to actually knock. +""" + +from __future__ import annotations + +import importlib +import shutil +import sys +from dataclasses import dataclass +from typing import Callable, Optional + +from . import config + +OK, WARN, FAIL = "ok", "warn", "fail" + +_MARKS = {OK: " ok ", WARN: " warn ", FAIL: " FAIL "} + + +@dataclass +class Check: + name: str + status: str + detail: str = "" + fix: str = "" + + def line(self) -> str: + text = f"[{_MARKS[self.status]}] {self.name:22} {self.detail}" + if self.fix and self.status != OK: + text += f"\n{'':32}→ {self.fix}" + return text + + +def _module(name: str) -> bool: + try: + importlib.import_module(name) + return True + except Exception: + return False + + +# ── the checks ────────────────────────────────────────────────────────────── + +def check_config() -> Check: + missing = config.missing_config() + if missing: + return Check("server config", FAIL, f"missing {', '.join(missing)}", + "set them in .env — without these the controller exits at startup") + return Check("server config", OK, f"{config.SERVER_URL} as {config.SESSION_ID}") + + +def check_server(deep: bool = False) -> Check: + if not config.SERVER_URL: + return Check("server", FAIL, "no BOLT_SERVER_URL", + "cp .env.example .env and set BOLT_SERVER_URL to your Bolt server") + if not deep: + return Check("server", OK, f"{config.SERVER_URL} (not contacted; --deep to try)") + try: + from . import server_client + + health = server_client.check_health(timeout=8) + return Check("server", OK, f"reachable — {health}") + except Exception as exc: + return Check("server", FAIL, f"unreachable: {exc}", + "check the URL, the key, and that the container is up") + + +def check_streaming_endpoint(deep: bool = False) -> Check: + """The streamed-reply endpoint is newer than some deployed servers.""" + if not config.STREAMING_REPLIES: + return Check("streamed replies", WARN, "disabled (STREAMING_REPLIES=false)") + if not deep: + return Check("streamed replies", OK, "enabled (endpoint not probed)") + try: + import requests + + response = requests.post( + f"{config.SERVER_URL}/desk/converse_stream", + json={"session_id": config.SESSION_ID, "text": ""}, + headers={"X-Desk-Api-Key": config.API_KEY}, timeout=8, stream=True, + ) + if response.status_code == 404: + return Check("streamed replies", WARN, "server has no /desk/converse_stream", + "update the server, or set STREAMING_REPLIES=false to skip the probe") + return Check("streamed replies", OK, f"endpoint answered {response.status_code}") + except Exception as exc: + return Check("streamed replies", WARN, f"probe failed: {exc}") + + +def check_microphone(deep: bool = False) -> Check: + if not _module("sounddevice"): + return Check("microphone", FAIL, "sounddevice is not installed", + "pip install -r requirements.txt") + try: + import sounddevice as sd + + devices = [d for d in sd.query_devices() if d.get("max_input_channels", 0) > 0] + if not devices: + return Check("microphone", FAIL, "no input devices", + "on Linux check PipeWire/PulseAudio is running as your user") + chosen = config.MIC_DEVICE or "system default" + if not deep: + return Check("microphone", OK, f"{len(devices)} input device(s), using {chosen}") + with sd.InputStream(samplerate=config.SAMPLE_RATE, channels=1, dtype="int16", + device=config.MIC_DEVICE, blocksize=config.FRAME_LEN): + pass + return Check("microphone", OK, f"opened at {config.SAMPLE_RATE} Hz ({chosen})") + except Exception as exc: + return Check("microphone", FAIL, f"could not open: {exc}", + "don't run the pet as root — PortAudio can't reach your PipeWire socket") + + +def check_wake_model() -> Check: + from pathlib import Path + + path = Path(config.WAKE_MODEL_PATH) + if not path.exists(): + return Check("wake word", FAIL, f"{path.name} is missing", + "it ships in the project root; check WAKE_MODEL_FILE") + if not _module("openwakeword"): + return Check("wake word", FAIL, "openwakeword is not installed", + "pip install -r requirements.txt") + return Check("wake word", OK, f"{path.name}, threshold {config.WAKE_WORD_THRESHOLD}") + + +def check_stt() -> Check: + if not config.DEEPGRAM_API_KEY: + return Check("speech-to-text", FAIL, "no DEEPGRAM_API_KEY", + "set it in .env — nothing you say can be transcribed without it") + if config.STT_STREAMING and not _module("websocket"): + return Check("speech-to-text", WARN, "streaming on, but websocket-client is missing", + "pip install websocket-client — it falls back to one-shot uploads") + mode = "streaming" if config.STT_STREAMING else "one-shot" + return Check("speech-to-text", OK, f"Deepgram {config.DEEPGRAM_MODEL}, {mode}") + + +def check_tts() -> Check: + if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID): + if _module("pyttsx3"): + return Check("text-to-speech", WARN, "no ElevenLabs key/voice — offline voice only", + "set ELEVENLABS_API_KEY and ELEVENLABS_VOICE_ID for the real voice") + return Check("text-to-speech", FAIL, "no ElevenLabs config and no pyttsx3 fallback") + return Check("text-to-speech", OK, + f"ElevenLabs {config.ELEVENLABS_MODEL_ID}, voice …{config.ELEVENLABS_VOICE_ID[-6:]}") + + +def check_dialogue() -> Check: + if not config.DIALOGUE: + return Check("multi-voice scenes", WARN, "disabled (DIALOGUE=false)") + from . import dialogue + + cast = dialogue.parse_voice_map(config.DIALOGUE_VOICES) + if not cast: + return Check("multi-voice scenes", WARN, "no cast configured — only 'self' works", + 'set DIALOGUE_VOICES=narrator:,villain:') + return Check("multi-voice scenes", OK, f"{len(cast)} voice(s): {', '.join(sorted(cast))}") + + +def check_screen_text() -> Check: + if not config.SCREEN_TEXT: + return Check("screen reading", WARN, "disabled (SCREEN_TEXT=false)") + if not _module("mss"): + return Check("screen reading", WARN, "mss is not installed — petctl read will decline", + "pip install mss (and note it cannot capture on Wayland)") + engine = "pytesseract" if _module("pytesseract") else ( + "rapidocr" if _module("rapidocr_onnxruntime") else "") + if not engine: + return Check("screen reading", WARN, "no OCR engine", + "pip install pytesseract && apt install tesseract-ocr, " + "or pip install rapidocr-onnxruntime") + if engine == "pytesseract" and not shutil.which("tesseract"): + return Check("screen reading", WARN, "pytesseract is installed but tesseract is not", + "apt install tesseract-ocr") + return Check("screen reading", OK, f"mss + {engine}") + + +def check_hotkey() -> Check: + if not _module("pynput"): + return Check("push-to-talk", WARN, "pynput is not installed", + "the wake word still works; pip install pynput for the hotkey") + import os + + if os.environ.get("WAYLAND_DISPLAY") and not os.environ.get("DISPLAY"): + return Check("push-to-talk", WARN, "Wayland session — global hotkeys usually blocked", + "use the wake word, or click the pet") + return Check("push-to-talk", OK, config.PUSH_TO_TALK_HOTKEY) + + +def check_sprites() -> Check: + from pathlib import Path + + root = Path(__file__).resolve().parent / "assets" / "sprites" + if not root.exists(): + return Check("sprites", WARN, "no art — the placeholder blob will be drawn", + "python scripts/generate_bolt_sprites.py") + counts = {d.name: len(list(d.glob("*.png"))) for d in sorted(root.iterdir()) if d.is_dir()} + empty = [name for name, count in counts.items() if count == 0] + if empty: + return Check("sprites", WARN, f"no frames for: {', '.join(empty)}", + "python scripts/generate_bolt_sprites.py") + return Check("sprites", OK, ", ".join(f"{n} {c}" for n, c in counts.items())) + + +def check_latency() -> Check: + """The setting most likely to make it feel slow, and the least obvious.""" + silence = config.SILENCE_END_SEC + if silence >= 1.5: + return Check("turn latency", WARN, f"VAD_SILENCE_END_SEC={silence:g}s of dead air per turn", + "0.8-1.0 feels markedly snappier; it is pure wait before anything starts") + return Check("turn latency", OK, f"silence timeout {silence:g}s, " + f"streaming {'on' if config.STREAMING_REPLIES else 'off'}") + + +CHECKS: tuple[tuple[str, Callable], ...] = ( + ("config", check_config), + ("server", check_server), + ("stream", check_streaming_endpoint), + ("mic", check_microphone), + ("wake", check_wake_model), + ("stt", check_stt), + ("tts", check_tts), + ("dialogue", check_dialogue), + ("screen", check_screen_text), + ("hotkey", check_hotkey), + ("sprites", check_sprites), + ("latency", check_latency), +) + + +def run(deep: bool = False) -> list[Check]: + results = [] + for _name, check in CHECKS: + try: + try: + results.append(check(deep)) + except TypeError: + results.append(check()) + except Exception as exc: # a broken check must not hide the others + results.append(Check(_name, FAIL, f"the check itself failed: {exc}")) + return results + + +def main(argv: Optional[list] = None) -> int: + argv = list(argv if argv is not None else sys.argv[1:]) + deep = "--deep" in argv + print(f"Bolt pet preflight{' (deep: contacting the server and opening the mic)' if deep else ''}\n") + results = run(deep=deep) + for check in results: + print(check.line()) + failures = [c for c in results if c.status == FAIL] + warnings = [c for c in results if c.status == WARN] + print() + if failures: + print(f"{len(failures)} problem(s) will stop this working. Fix those first.") + elif warnings: + print(f"Ready. {len(warnings)} thing(s) degraded but working.") + else: + print("Everything checks out.") + return 1 if failures else 0 diff --git a/bolt_pet/server_client.py b/bolt_pet/server_client.py index f0a60e1..5908f2b 100644 --- a/bolt_pet/server_client.py +++ b/bolt_pet/server_client.py @@ -19,6 +19,7 @@ Kept dependency-free beyond `requests` so it's easy to unit test with mocks. from __future__ import annotations +import json import subprocess from pathlib import Path from typing import Callable, NamedTuple, Optional @@ -43,6 +44,10 @@ class Reply(NamedTuple): text: str voice_id: str = "" voice_name: str = "" + # True when the sentences were already spoken as they streamed in. The + # text is still carried — the follow-up rule needs to see whether the + # answer ended on a question — it just must not be read out again. + spoken: bool = False def _headers() -> dict: @@ -90,12 +95,20 @@ def converse( text: str, on_command: Callable[[str], str] = run_local_command, timeout: float = 120.0, + on_say: Optional[Callable[[str], None]] = None, ) -> Reply: """Send one turn of conversation to the desk API, relaying any commands the server sends back until it produces a final reply. *on_command* is injectable for tests; defaults to actually running the command locally (matching bolt_desk.py's behavior). + + *on_say* is called with a short holding line ("give me a sec") when the + server sends one alongside a command. It is the difference between silence + and an answer while a tool runs: the model's acknowledgement used to be + discarded server-side, so the whole round trip was dead air and the model + then repeated itself in the final reply. Optional, so an older pet against + a newer server simply stays quiet as before. """ headers = _headers() try: @@ -111,6 +124,13 @@ def converse( for _ in range(_MAX_RELAY_HOPS): if payload.get("type") != "command": break + holding = str(payload.get("say") or "").strip() + if holding and on_say is not None: + # Spoken *before* the command runs — that is the whole point. + try: + on_say(holding) + except Exception: + pass # a failed acknowledgement must not cost the tool call output = on_command(str(payload.get("command") or "")) try: response = requests.post( @@ -135,6 +155,104 @@ def converse( raise ServerError(str(payload.get("error") or "unknown server response")) +def converse_stream( + text: str, + on_say: Callable[[str], None], + on_command: Callable[[str], str] = run_local_command, + timeout: float = 180.0, +) -> Reply: + """Same turn as converse(), but speaking each sentence as it arrives. + + Without this the pet waits out the *entire* model call before a single + word is heard; with it the wait is time-to-first-sentence, which on a + multi-sentence answer is most of the difference. + + Falls back by raising ServerError before anything has been spoken — the + caller then retries the ordinary path and the user never finds out. Once a + sentence *has* been spoken there is no going back, so late failures end the + turn with whatever was said rather than repeating it. + """ + spoke_anything = False + try: + response = requests.post( + f"{config.SERVER_URL}/desk/converse_stream", + json={"session_id": config.SESSION_ID, "text": text}, + headers=_headers(), timeout=timeout, stream=True, + ) + response.raise_for_status() + for raw in response.iter_lines(decode_unicode=True): + if not raw: + continue + try: + event = json.loads(raw) + except (TypeError, ValueError): + continue + kind = str(event.get("type") or "") + + if kind == "say": + line = str(event.get("text") or "").strip() + if line: + spoke_anything = True + on_say(line) + elif kind == "command": + # The tool loop is request/response, so the rest of the turn + # finishes through the ordinary relay rather than inside the + # stream — one protocol for tools, not two. + response.close() + return _finish_relay(event, on_command, on_say) + elif kind == "reply": + return Reply( + text=str(event.get("text") or ""), + voice_id=str(event.get("voice_id") or ""), + voice_name=str(event.get("voice_name") or ""), + spoken=bool(event.get("already_spoken")), + ) + elif kind == "error": + raise ServerError(str(event.get("error") or "stream failed")) + except ServerError: + raise + except Exception as exc: + if spoke_anything: + # Half a reply is out loud already; ending quietly beats saying it + # all again through the fallback path. + return Reply(text="", spoken=True) + raise ServerError(f"streaming failed: {exc}") from exc + raise ServerError("stream ended without a reply") + + +def _finish_relay(event: dict, on_command, on_say) -> Reply: + """Run the tool the stream handed over, then continue the classic relay.""" + payload = dict(event) + headers = _headers() + for _ in range(_MAX_RELAY_HOPS): + if payload.get("type") != "command": + break + holding = str(payload.get("say") or "").strip() + if holding and on_say is not None: + try: + on_say(holding) + except Exception: + pass + output = on_command(str(payload.get("command") or "")) + try: + response = requests.post( + f"{config.SERVER_URL}/desk/tool_result", + json={"session_id": config.SESSION_ID, + "token": payload.get("token"), "output": output}, + headers=headers, timeout=180, + ) + payload = response.json() + except Exception as exc: + raise ServerError(f"couldn't reach the server during tool relay: {exc}") from exc + if payload.get("type") == "reply": + return Reply( + text=str(payload.get("text") or ""), + voice_id=str(payload.get("voice_id") or ""), + voice_name=str(payload.get("voice_name") or ""), + ) + raise ServerError(str(payload.get("error") or "unknown server response")) + + def list_outbox_files(timeout: float = 15.0) -> list: """Files the server has queued for this session via its deliver_files tool (e.g. "send me that report" during a conversation) — each entry has diff --git a/bolt_pet/speech.py b/bolt_pet/speech.py new file mode 100644 index 0000000..83f229a --- /dev/null +++ b/bolt_pet/speech.py @@ -0,0 +1,194 @@ +"""Everything the pet says, and the policy differences between kinds of saying. + +There used to be four of these in controller.py — a reply, a holding line, a +streamed sentence, a dialogue scene — each written when its feature was built, +each repeating the same dance: transition state, show the bubble, maybe record +history, reset barge-in, call TTS, reset barge-in again, resume state, decide +whether to keep the mic open. Only the *policy* differed, and the copies had +already started to drift: one forgot to arm barge-in, another logged a +different prefix, a third skipped the follow-up rule. + +So the dance lives here once, and the differences are data: + + reply the answer. Transcript, bubble, follow-up rule, ends IDLE. + holding "give me a sec" while a tool runs. No transcript — it is + filler, and the transcript should keep the answer. Resumes + whatever state it interrupted, because the turn isn't over. + stream one sentence of a streamed answer. Transcript and bubble like + a reply, but stays TALKING so the sprite doesn't flicker + between sentences, and the follow-up rule waits for the last. + scene a dialoguectl take. Transcript (the user heard it), resumes + mid-turn like a holding line. + +The controller keeps the pipeline; this keeps the rules about talking. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Optional + +from . import config, speech_text +from .state import PetState + + +@dataclass(frozen=True) +class Utterance: + """One thing to say, and how saying it should behave.""" + + text: str + record: bool = True # goes in the transcript, or is it filler? + resume: bool = False # return to the state it interrupted (mid-turn) + hold_talking: bool = False # stay TALKING afterwards (more is coming) + follow_up: bool = True # may leave the mic open if it ends on a question + interruptible: bool = True # arm barge-in for this one + log_prefix: str = "Bolt" + + @classmethod + def reply(cls, text: str) -> "Utterance": + return cls(text) + + @classmethod + def holding(cls, text: str) -> "Utterance": + # Filler: no transcript, no follow-up, and not interruptible — cutting + # off "give me a sec" would strand the tool that is already running. + return cls(text, record=False, resume=True, follow_up=False, + interruptible=False, log_prefix="Bolt (holding)") + + @classmethod + def stream_chunk(cls, text: str) -> "Utterance": + return cls(text, hold_talking=True, follow_up=False) + + @classmethod + def scene(cls, text: str) -> "Utterance": + return cls(text, resume=True, follow_up=False) + + +class Speaker: + """Says things on behalf of the controller. + + Collaborators are passed in rather than reached for, so the whole of + speaking is testable without Qt, audio hardware or a state machine: hand it + fakes and assert on what came out.""" + + def __init__(self, *, state, tts, history=None, on_said=None, on_log=None, + barge_in: Callable[[], object] = lambda: None, + voice_id: Callable[[], str] = lambda: "", + detail_of: Callable[[], str] = lambda: "", + on_level: Optional[Callable[[float], None]] = None): + self._state = state + self._tts = tts + self._history = history + self._on_said = on_said or (lambda _text: None) + self._on_log = on_log or (lambda _msg: None) + # Read through a callable rather than held: the detector is built after + # the speaker (it needs the mic stream), replaced when barge-in mode + # changes, and swapped by tests. Two copies of it drifting apart is a + # bug nobody notices until the wake model starts hearing the pet. + self._barge_in_of = barge_in + self._voice_id = voice_id + # What actually fired, captured BEFORE the post-playback reset. Read it + # afterwards and every interruption reports 0.000 at frame 0 — which + # looks like hard evidence and is nothing of the sort. + self._detail_of = detail_of + self.last_detail = "" + self._on_level = on_level + + @property + def _barge_in(self): + try: + return self._barge_in_of() + except Exception: + return None + + def say(self, utterance: Utterance) -> bool: + """Speak it. Returns False if it was interrupted. + + The one place that knows the order these steps go in — which is the + point, because getting that order wrong is invisible until the wake + model starts hearing the pet's own voice.""" + line = speech_text.for_display(utterance.text) + if not line: + return True + + resume_state = self._state.state + if self._state.state != PetState.TALKING: + self._state.transition(PetState.TALKING) + self._on_said(line) + self._on_log(f"{utterance.log_prefix}: {line}") + if utterance.record and self._history is not None: + self._history(utterance.text) + + should_stop = None + if self._barge_in is not None and utterance.interruptible: + self._barge_in.reset() + should_stop = self._barge_in.check + + completed = self._tts.speak( + utterance.text, + on_error=lambda exc: self._on_log(f"TTS failed: {exc}"), + should_stop=should_stop, + voice_id=self._voice_id() or None, + on_level=self._on_level, + ) + self.last_detail = self._detail_of() + if self._barge_in is not None: + # Playback fed the pet's own voice into the wake model's rolling + # window. Clear it before the idle listener scores again, or the + # last sentence is still in there being re-heard. + self._barge_in.reset() + if self._on_level is not None: + self._on_level(0.0) # mouth closed; nothing is playing now + + if utterance.resume and resume_state in (PetState.THINKING, PetState.IDLE): + self._state.transition(resume_state) + return bool(completed) + + + def say_pcm(self, utterance: Utterance, *, pcm, sample_rate: int) -> bool: + """Speak audio that is already synthesized — a dialoguectl scene. + + Same policy, same barge-in handling, same mouth; only the source of + the samples differs. Sharing this is why a scene can be talked over + exactly like an ordinary reply.""" + line = speech_text.for_display(utterance.text) + resume_state = self._state.state + if self._state.state != PetState.TALKING: + self._state.transition(PetState.TALKING) + if line: + self._on_said(line) + self._on_log(f"{utterance.log_prefix}: {line}") + if utterance.record and self._history is not None: + self._history(utterance.text) + + should_stop = None + if self._barge_in is not None and utterance.interruptible: + self._barge_in.reset() + should_stop = self._barge_in.check + completed = self._tts.play_pcm( + pcm, sample_rate, should_stop=should_stop, on_level=self._on_level) + self.last_detail = self._detail_of() + if self._barge_in is not None: + self._barge_in.reset() + if self._on_level is not None: + self._on_level(0.0) + if utterance.resume and resume_state in (PetState.THINKING, PetState.IDLE): + self._state.transition(resume_state) + return bool(completed) + + +def follow_up_decision(text: str, *, completed: bool, follow_ups: int) -> tuple[bool, str]: + """Whether to keep listening after speaking, and why. + + Split out as a pure function because it is a *rule* with an off-by-one cap + in it, and rules with counters deserve a test that doesn't need audio.""" + if not completed: + return True, "interrupted" + if not speech_text.is_question(text): + return False, "" + cap = config.FOLLOW_UP_MAX_TURNS + if not config.FOLLOW_UP_LISTEN: + return False, "" + if cap > 0 and follow_ups >= cap: + return False, f"follow-up cap ({cap}) reached" + return True, "question" diff --git a/bolt_pet/speech_text.py b/bolt_pet/speech_text.py index 5aa2b13..fd82242 100644 --- a/bolt_pet/speech_text.py +++ b/bolt_pet/speech_text.py @@ -97,10 +97,34 @@ def _bullets_to_sentences(text: str) -> str: return " ".join(line if line[-1] in ".!?:,;" else line + "." for line in lines) +# ElevenLabs v3 delivery tags — "[laughing]", "[whispering]", "[sighs]". They +# are instructions to the *dialogue* model (see dialogue.py), and only inside a +# dialoguectl scene. Left in an ordinary reply they reach eleven_flash_v2, +# which has no idea what they mean and simply reads the word: observed live +# 2026-07-31, the pet announcing "laughing That one came through clean". +# +# Matched narrowly — a bracketed adverb/gerund, or one of the noise words that +# aren't either — so real bracketed text ("[1]", "[see the docs]") survives. +_AUDIO_TAG_RE = re.compile( + r"\[\s*(?:" + r"[a-z]+(?:ly|ing)" + r"|laughs?|chuckles?|sighs?|exhales?|inhales?|gulps?|pause|beat" + r"|clears throat|shouts?|whispers?|cries|sings?|gasps?|snorts?" + r"|sarcastic|excited|curious|nervous|angry|sad|happy|deadpan|flat" + r")\s*\]", + re.IGNORECASE, +) + + +def strip_audio_tags(text: str) -> str: + """Remove v3 delivery tags from text destined for the ordinary voice.""" + return _AUDIO_TAG_RE.sub(" ", str(text or "")) + + def for_speech(text: str) -> str: """Plain prose for the TTS engine: no markdown, no emoji, no bare URLs, no stray symbols that would be read out character by character.""" - text = (text or "").strip() + text = strip_audio_tags((text or "").strip()) if not text: return "" for symbol, spoken in _PRE_SPOKEN_SYMBOLS.items(): @@ -136,8 +160,11 @@ def is_question(text: str) -> bool: def for_display(text: str) -> str: """What the speech bubble shows: markdown syntax removed (the bubble - can't render it) but emoji and layout-ish punctuation left alone.""" - text = (text or "").strip() + can't render it) but emoji and layout-ish punctuation left alone. + + Delivery tags go too — the voice no longer says them, so showing them + would caption a laugh nobody heard.""" + text = strip_audio_tags((text or "").strip()) if not text: return "" text = _strip_markdown(text, keep_emoji=True) diff --git a/bolt_pet/ui/app.py b/bolt_pet/ui/app.py index b575c16..7725858 100644 --- a/bolt_pet/ui/app.py +++ b/bolt_pet/ui/app.py @@ -41,6 +41,8 @@ def run() -> int: thread.started.connect(controller.run) controller.state_changed.connect(lambda value: window.set_state(PetState(value))) controller.said.connect(window.say) + # Lip-sync: the loudness of the audio actually going to the speakers. + controller.mouth.connect(window.set_mouth) controller.log.connect(_log) controller.action.connect(window.apply_action) # petctl move/emote/say/... controller.finished.connect(thread.quit) diff --git a/bolt_pet/ui/pet_window.py b/bolt_pet/ui/pet_window.py index 075813e..8bce0bf 100644 --- a/bolt_pet/ui/pet_window.py +++ b/bolt_pet/ui/pet_window.py @@ -19,7 +19,7 @@ from PySide6.QtGui import ( ) from PySide6.QtWidgets import QApplication, QWidget -from .. import config +from .. import config, window_state from ..monitors import Monitor from ..state import PetState from .sprite import WALK, SpriteSet @@ -36,6 +36,11 @@ _NAP_OPACITY = 0.35 # and the feet skate whenever PET_WANDER_SPEED doesn't happen to match the fps. # Eight frames at 13px is a ~104px stride cycle, a bit under the pet's width. _WALK_PIXELS_PER_FRAME = 13.0 +# How long a loudness level stays believable. The audio thread sends one per +# ~30ms while a clip plays; if they stop arriving (offline TTS has no envelope, +# or playback died) the mouth must not stay frozen mid-syllable, so after this +# long the ordinary looping animation takes back over. +_MOUTH_STALE_SECONDS = 0.35 def emote_transform(emote: str, progress: float) -> tuple[float, float, float, float]: @@ -213,6 +218,9 @@ class PetWindow(QWidget): self._commanded_move = False # a petctl move — happens even mid-conversation self._next_wander_at = 0.0 self._bob_offset = 0 + # Lip-sync: loudness of what is playing right now, and when it arrived. + self._mouth_level: Optional[float] = None + self._mouth_at = 0.0 self._bob_phase = 0.0 self._walking = False self._facing = 1 # +1 right, -1 left; the walk art is drawn facing right @@ -249,6 +257,19 @@ class PetWindow(QWidget): y = int(config.PET_START_Y) if config.PET_START_Y else None except ValueError: x = y = None + if x is None and y is None and config.PET_REMEMBER_POSITION: + remembered = window_state.load() + # Only if it still lands on a screen that exists — the usual reason + # a saved position is stale is that the monitor it was on has been + # unplugged, and restoring it faithfully would hide the pet. + if remembered is not None: + rectangles = [ + (s.availableGeometry().left(), s.availableGeometry().top(), + s.availableGeometry().right(), s.availableGeometry().bottom()) + for s in QApplication.screens() + ] + if window_state.is_visible_on(*remembered, self.width(), rectangles): + x, y = remembered if geo is not None: x = geo.right() - self.width() - 40 if x is None else x y = geo.bottom() - self.height() - 60 if y is None else y @@ -688,6 +709,34 @@ class PetWindow(QWidget): # ── animation ──────────────────────────────────────────────────────── + def set_mouth(self, level: float) -> None: + """How loud the pet is *right now* (0..1), straight off the PCM going + to the speakers (audio/tts.level_of). + + The talking frames are ordered by mouth openness, so this indexes them + directly: the mouth moves with the actual waveform instead of flapping + on a timer, which is the difference between a talking sprite and a + dubbed one.""" + self._mouth_level = max(0.0, min(1.0, float(level))) + self._mouth_at = time.monotonic() + if self._current_state == PetState.TALKING: + self.update() + + def _mouth_frame(self, animation) -> Optional[QPixmap]: + """The frame matching the current loudness, or None to use the timer. + + Falls back the moment the levels go stale — offline TTS has no + envelope, and a mouth frozen mid-syllable is worse than a timed loop.""" + if self._mouth_level is None or animation is None: + return None + if time.monotonic() - self._mouth_at > _MOUTH_STALE_SECONDS: + return None + frames = animation.frames + if len(frames) < 2: + return None + index = int(round(self._mouth_level * (len(frames) - 1))) + return frames[max(0, min(len(frames) - 1, index))] + def _advance_frame(self) -> None: # While walking the cycle is stepped by _advance_walk from distance # travelled; letting this timer also advance it would double-step it @@ -702,7 +751,12 @@ class PetWindow(QWidget): painter.setRenderHint(QPainter.Antialiasing) painter.setRenderHint(QPainter.SmoothPixmapTransform) key = self._animation_key() - pixmap: Optional[QPixmap] = self.sprites.get(key).current() + animation = self.sprites.get(key) + pixmap: Optional[QPixmap] = None + if key == PetState.TALKING: + pixmap = self._mouth_frame(animation) + if pixmap is None: + pixmap = animation.current() if key == WALK: pixmap = self._oriented(pixmap) if pixmap is None: @@ -759,6 +813,9 @@ class PetWindow(QWidget): was_click = not self._dragged self._drag_offset = None self._press_pos = None + if not was_click and config.PET_REMEMBER_POSITION: + position = self.geometry().topLeft() + window_state.save(position.x(), position.y()) if was_click: self.talk_requested.emit() else: diff --git a/bolt_pet/window_state.py b/bolt_pet/window_state.py new file mode 100644 index 0000000..ca45fc4 --- /dev/null +++ b/bolt_pet/window_state.py @@ -0,0 +1,75 @@ +"""Where the pet was left, so it starts there next time. + +Listed in the README as a known limitation: drag it somewhere deliberate, and +the next launch puts it back in the bottom-right corner. For something that +lives on your desktop all day that is a small daily annoyance, and it is a +config write on drag-end. + +Lives in the cache dir rather than the repo, next to the restart context: it +is per-machine state about this install, not something that belongs in a git +diff. Every function swallows its own errors — a corrupt or unwritable state +file must never stop the pet from starting, it just means the default corner. + +Positions are validated against the *current* screen layout on load, because +the common case for a stale position is exactly the case where it is +dangerous: the pet was last on a monitor that is now unplugged, and restoring +it faithfully would put it somewhere you cannot see or reach. +""" + +from __future__ import annotations + +import json +import logging +import os +import tempfile +from pathlib import Path +from typing import Optional + +logger = logging.getLogger("bolt_pet.window_state") + +DEFAULT_PATH = Path.home() / ".cache" / "bolt-pet" / "window.json" + + +def load(path: Optional[Path] = None) -> Optional[tuple[int, int]]: + """The saved position, or None if there isn't a usable one.""" + try: + data = json.loads(Path(path or DEFAULT_PATH).read_text(encoding="utf-8")) + return int(data["x"]), int(data["y"]) + except (FileNotFoundError, json.JSONDecodeError, KeyError, TypeError, ValueError, OSError): + return None + + +def save(x: int, y: int, path: Optional[Path] = None) -> None: + """Remember where it is now. Atomic, so a crash mid-write can't leave a + half-file that makes the next start fall back to the corner.""" + target = Path(path or DEFAULT_PATH) + try: + target.parent.mkdir(parents=True, exist_ok=True) + descriptor, temp_path = tempfile.mkstemp(dir=target.parent, prefix=".window_", suffix=".tmp") + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump({"x": int(x), "y": int(y)}, handle) + os.replace(temp_path, target) + except BaseException: + try: + os.unlink(temp_path) + except OSError: + pass + raise + except Exception: + logger.debug("Could not save the window position", exc_info=True) + + +def is_visible_on(x: int, y: int, size: int, rectangles) -> bool: + """Whether that position still lands on a screen that exists. + + *rectangles* are (left, top, right, bottom) tuples — the caller's job, + because this module has no business importing Qt. Requires a real overlap + rather than a touching edge, so a pet saved flush against the boundary of a + monitor that has since been unplugged is not counted as reachable.""" + for left, top, right, bottom in rectangles: + overlap_x = min(x + size, right) - max(x, left) + overlap_y = min(y + size, bottom) - max(y, top) + if overlap_x > size * 0.25 and overlap_y > size * 0.25: + return True + return False diff --git a/requirements.txt b/requirements.txt index 54b6124..9329ebb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,9 @@ numpy>=1.24 # HTTP client to the Bolt desk API requests>=2.31 +# Streaming speech-to-text (audio/stt_stream.py). Optional in practice: without +# it the pet falls back to uploading the finished clip, exactly as before. +websocket-client>=1.7 # Wake-word detection (local, offline after first run) — runs the # custom-trained thunderbolt.onnx model shipped in this repo, same runtime diff --git a/scripts/generate_bolt_sprites.py b/scripts/generate_bolt_sprites.py index ea0f186..53b8ea5 100644 --- a/scripts/generate_bolt_sprites.py +++ b/scripts/generate_bolt_sprites.py @@ -663,31 +663,43 @@ def render_frame(p) -> Image.Image: def frames_for(state: str) -> list[dict]: if state == "idle": + # Eight frames, all distinct. The old version drove breathing on + # sin(2*pi*t) and the tail on sin(4*pi*t), which both cross zero at + # i=0 and i=4 — so frame 4 was byte-identical to frame 0 and the loop + # was really four frames stored twice. out = [] for i in range(8): t = i / 8 - br = math.sin(t * 2 * math.pi) + br = math.sin(t * 2 * math.pi + math.pi / 7) out.append( default_pose( breathe=br, head_dy=-0.006 * br, - tail=math.sin(t * 4 * math.pi), + # Three-halves harmonic: never in phase with the breath, so + # no two frames of the cycle can coincide. + tail=math.sin(t * 3 * math.pi + 0.6), + ear_twitch=0.30 if i == 3 else 0.0, # a flick, once a loop blink=1.0 if i == 6 else 0.0, ) ) return out if state == "listening": + # Six frames of *orienting*, not idling: ears up, head turning toward + # whoever is talking, then settling. The old four had frames 0 and 2 + # differing by 0.09 mean pixels — a two-pose animation wearing four. out = [] - for i in range(4): - t = i / 4 + for i in range(6): + t = i / 6 + lean = math.sin(t * 2 * math.pi + math.pi / 5) out.append( default_pose( ear=1.0, - ear_twitch=0.35 * math.sin(t * 2 * math.pi), - tilt=-7 + 2.0 * math.sin(t * 2 * math.pi), + ear_twitch=0.45 * math.sin(t * 4 * math.pi), + tilt=-9 + 4.0 * lean, + look=(0.010 * lean, -0.004), brow=1.0, - tail=0.5 * math.sin(t * 2 * math.pi), - head_dy=-0.008, + tail=0.7 * math.sin(t * 2 * math.pi + 1.1), + head_dy=-0.010 - 0.004 * lean, tag_glow=True, extras="listen", phase=i, @@ -695,35 +707,51 @@ def frames_for(state: str) -> list[dict]: ) return out if state == "thinking": + # The old six moved by a mean of ~1.3 pixels — effectively a still + # image. Thinking should *look* like thinking: the head tilts, the eyes + # travel as if following a thought, and one ear rotates independently. out = [] for i in range(6): t = i / 6 + sway = math.sin(t * 2 * math.pi) out.append( default_pose( - ear=0.25, - tilt=6.0, - look=(0.022, -0.026), - brow=0.5, - breathe=0.4 * math.sin(t * 2 * math.pi), - tail=0.2 * math.sin(t * 2 * math.pi), + ear=0.25 + 0.35 * abs(sway), + ear_twitch=0.5 * math.cos(t * 2 * math.pi), + tilt=4.0 + 7.0 * sway, + # Eyes wander a small circle: the cheapest possible read of + # "working something out" and the thing most obviously + # missing before. + look=(0.026 * math.cos(t * 2 * math.pi), + -0.020 + 0.014 * math.sin(t * 2 * math.pi)), + brow=0.5 + 0.4 * abs(sway), + breathe=0.5 * math.sin(t * 2 * math.pi + 0.9), + head_dy=-0.008 * sway, + tail=0.35 * math.sin(t * 3 * math.pi), + blink=1.0 if i == 4 else 0.0, extras="think", phase=i // 2, ) ) return out if state == "talking": + # Ordered by mouth openness — closed at frame 0, widest at the last — + # because the window indexes these by the loudness of the audio that is + # actually playing (see audio/tts.level_of and PetWindow.set_mouth). + # A time-ordered loop cannot be indexed that way, and a mouth that + # flaps on a timer is what makes a talking sprite look dubbed. out = [] - for i in range(4): - t = i / 4 - open_ = (math.sin(t * 2 * math.pi) + 1) / 2 + count = 6 + for i in range(count): + open_ = i / (count - 1) out.append( default_pose( - mouth=0.25 + 0.75 * open_, + mouth=0.06 + 0.94 * open_, ear=0.6, head_dy=-0.010 * open_, breathe=open_, - tail=math.sin(t * 2 * math.pi + 1.0), - brow=0.35, + tail=0.45 * math.sin(i * 0.9), + brow=0.35 * open_, ) ) return out diff --git a/tests/test_controller.py b/tests/test_controller.py index 52da051..b4a0322 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -22,6 +22,14 @@ def no_screen_probes(monkeypatch): monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False) +@pytest.fixture(autouse=True) +def _no_streaming(monkeypatch): + """Most tests drive the plain request/response path; leaving streaming on + would have them attempt a real HTTP call, fail, and fall back — passing, + slowly, for the wrong reason. The streaming path has its own tests.""" + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False) + + @pytest.fixture def ctrl(): return controller_mod.PetController() @@ -41,10 +49,10 @@ def test_full_turn_happy_path(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16)) monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's the weather") - monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None: controller_mod.server_client.Reply("sunny and 72")) + monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None, on_say=None: controller_mod.server_client.Reply("sunny and 72")) spoken = [] monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1]) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1]) ctrl._handle_conversation_turn() @@ -59,7 +67,7 @@ def test_turn_with_nothing_heard_returns_to_idle_without_calling_server(monkeypa monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: None) called = {"n": 0} monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: called.__setitem__("n", called["n"] + 1)) + lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1)) ctrl._handle_conversation_turn() @@ -97,7 +105,7 @@ def test_turn_with_server_error_flashes_error_then_idle(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16)) monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "hello") - def boom(text, on_command=None): + def boom(text, on_command=None, on_say=None): raise controller_mod.server_client.ServerError("server is down") monkeypatch.setattr(controller_mod.server_client, "converse", boom) @@ -150,7 +158,7 @@ def test_heartbeat_speaks_a_pending_announcement_when_idle(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: "don't forget your 3pm") spoken = [] monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1]) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1]) said = _capture(ctrl.said) ctrl._maybe_heartbeat() diff --git a/tests/test_controller_features.py b/tests/test_controller_features.py index f76c674..fb4c4df 100644 --- a/tests/test_controller_features.py +++ b/tests/test_controller_features.py @@ -29,6 +29,14 @@ def no_screen_probes(monkeypatch): monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text) +@pytest.fixture(autouse=True) +def _no_streaming(monkeypatch): + """Most tests drive the plain request/response path; leaving streaming on + would have them attempt a real HTTP call, fail, and fall back — passing, + slowly, for the wrong reason. The streaming path has its own tests.""" + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False) + + @pytest.fixture def ctrl(): return controller_mod.PetController() @@ -160,7 +168,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch ctrl._barge_in = detector logs = _capture(ctrl.log) - def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None): + def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None, on_level=None): # What really happens: frames get scored during playback, then one # clears the threshold and playback aborts. detector._frames, detector._peak, detector._last = 7, 0.81, 0.81 @@ -180,7 +188,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl): logs = _capture(ctrl.log) monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: False) # interrupted + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False) # interrupted ctrl._speak("a very long explanation") @@ -190,7 +198,7 @@ def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl): def test_uninterrupted_playback_does_not_queue_a_turn(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) ctrl._speak("short answer") assert not ctrl._talk_now.is_set() @@ -202,7 +210,7 @@ def spoke(monkeypatch): """Playback that always completes, so only the follow-up rule decides whether another turn is queued.""" monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) def test_a_reply_ending_in_a_question_keeps_listening(spoke, ctrl): @@ -290,7 +298,7 @@ def test_follow_up_can_be_turned_off(spoke, monkeypatch, ctrl): def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: False) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False) ctrl._follow_ups = 3 ctrl._speak("a very long explanation") @@ -301,7 +309,7 @@ def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl): def test_speech_is_recorded_in_the_history(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) ctrl._speak("**bold** reply") assert ctrl.history.last().text == "**bold** reply" # raw, for copy/paste @@ -315,10 +323,10 @@ def test_the_active_window_rides_along_with_the_utterance(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: f"{text}\n\n[on screen right now: app.py]") monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) sent = [] monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: sent.append(text) or Reply("that's a KeyError")) + lambda text, on_command=None, on_say=None: sent.append(text) or Reply("that's a KeyError")) ctrl._handle_conversation_turn() @@ -371,10 +379,10 @@ def test_napping_still_answers_when_spoken_to(monkeypatch, ctrl): lambda *a, **k: np.zeros(10, dtype=np.int16)) monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "you awake?") monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: Reply("always")) + lambda text, on_command=None, on_say=None: Reply("always")) spoken = [] monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True) ctrl.set_napping(True) ctrl._handle_conversation_turn() @@ -389,9 +397,9 @@ def test_notifications_are_forwarded_and_spoken(monkeypatch, ctrl): ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) sent, spoken = [], [] monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: sent.append(text) or Reply("your build is green")) + lambda text, on_command=None, on_say=None: sent.append(text) or Reply("your build is green")) monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True) ctrl._queue_notification(Notification(app="CI", summary="Build finished", body="")) ctrl._drain_notifications() @@ -411,7 +419,7 @@ def test_notifications_are_not_forwarded_while_napping(monkeypatch, ctrl): ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) called = {"n": 0} monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: called.__setitem__("n", called["n"] + 1)) + lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1)) ctrl.set_napping(True) ctrl._queue_notification(Notification(app="CI", summary="Build finished", body="")) @@ -507,9 +515,9 @@ def test_check_deliveries_runs_after_a_conversation_turn(monkeypatch, ctrl, tmp_ lambda *a, **k: np.zeros(10, dtype=np.int16)) monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "send me that file") monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: Reply("it's on the way")) + lambda text, on_command=None, on_say=None: Reply("it's on the way")) monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: True) + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) monkeypatch.setattr(controller_mod.config, "DELIVERED_FILES_DIR", tmp_path) monkeypatch.setattr(controller_mod.server_client, "list_outbox_files", lambda: [{"id": "abc", "name": "notes.txt", "size": 2}]) @@ -531,10 +539,10 @@ def _voice_turn(monkeypatch, ctrl, reply, said="talk like a pirate"): lambda *a, **k: np.zeros(10, dtype=np.int16)) monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: said) monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: reply) + lambda text, on_command=None, on_say=None: reply) monkeypatch.setattr( controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: voices.append(voice_id) or True, ) ctrl._handle_conversation_turn() @@ -625,7 +633,7 @@ def test_dialoguectl_never_reaches_the_shell(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000)) monkeypatch.setattr(controller_mod.tts, "play_pcm", - lambda pcm, rate, should_stop=None: True) + lambda pcm, rate, should_stop=None, on_level=None: True) output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "[cheerfully] hi"})) @@ -638,7 +646,7 @@ def test_a_scene_shows_in_the_bubble_with_the_delivery_tags_stripped(monkeypatch monkeypatch.setattr(controller_mod.config, "DIALOGUE_VOICES", "narrator:9BWtsMINqrJLrRacOk9x") monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000)) - monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True) + monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True) said = _capture(ctrl.said) ctrl._handle_command(_dialogue_command( @@ -656,7 +664,7 @@ def test_a_mid_turn_scene_returns_to_thinking_not_idle(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.config, "ELEVENLABS_VOICE_ID", "aaorr6ZHIL88gEexu7dC") monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000)) - monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True) + monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True) ctrl._state.transition(PetState.LISTENING) ctrl._state.transition(PetState.THINKING) states = _capture(ctrl.state_changed) @@ -676,7 +684,7 @@ def test_the_scene_uses_a_voice_the_server_picked_with_speak_as(monkeypatch, ctr return np.zeros(4, dtype=np.int16), 24000 monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", capture) - monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True) + monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True) ctrl._apply_voice(controller_mod.server_client.Reply("ok", "PICKEDvoice123456789", "Terence")) ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"})) @@ -718,7 +726,7 @@ def test_talking_over_a_scene_is_reported_up_the_relay(monkeypatch, ctrl): monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000)) monkeypatch.setattr(controller_mod.tts, "play_pcm", - lambda pcm, rate, should_stop=None: False) # barge-in + lambda pcm, rate, should_stop=None, on_level=None: False) # barge-in output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"})) @@ -788,9 +796,9 @@ def test_coming_back_up_reports_to_the_server_and_speaks_the_reply(monkeypatch, path=state, now=1000.0) sent, spoken = [], [] monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: sent.append(text) or Reply("Good, it's up.")) + lambda text, on_command=None, on_say=None: sent.append(text) or Reply("Good, it's up.")) monkeypatch.setattr(controller_mod.tts, "speak", - lambda text, on_error=None, should_stop=None, voice_id=None: + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True) ctrl._report_self_restart() @@ -805,8 +813,230 @@ def test_an_ordinary_start_reports_nothing(monkeypatch, ctrl, tmp_path): monkeypatch.setattr(controller_mod.self_restart, "DEFAULT_STATE_PATH", tmp_path / "none.json") called = [] monkeypatch.setattr(controller_mod.server_client, "converse", - lambda text, on_command=None: called.append(text)) + lambda text, on_command=None, on_say=None: called.append(text)) ctrl._report_self_restart() assert called == [] + + +# ── holding line: "give me a sec" while a tool runs ──────────────────────── +# The server used to discard whatever the model wrote alongside a tool call, +# so the whole round trip was silence — and the model, with no evidence its +# sentence landed, said it again in the final reply. + +def test_a_holding_line_is_spoken_before_the_tool_runs(monkeypatch, ctrl): + order = [] + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: + order.append(("spoke", text)) or True) + monkeypatch.setattr(controller_mod.server_client, "run_local_command", + lambda cmd: order.append(("ran", cmd)) or "[exit 0]") + + ctrl._speak_holding("Give me a sec.") + ctrl._handle_command("df -h /") + + assert order == [("spoke", "Give me a sec."), ("ran", "df -h /")] + + +def test_the_holding_line_shows_in_the_bubble_but_not_the_transcript(monkeypatch, ctrl): + """It's filler. The transcript should keep the actual answer.""" + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) + said = _capture(ctrl.said) + + ctrl._speak_holding("I'll check on that — one moment.") + + assert said == ["I'll check on that — one moment."] + assert ctrl.history.entries() == [] + + +def test_a_holding_line_returns_to_waiting_not_idle(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) + ctrl._state.transition(PetState.LISTENING) + ctrl._state.transition(PetState.THINKING) + states = _capture(ctrl.state_changed) + + ctrl._speak_holding("one sec") + + assert states == ["talking", "thinking"] + + +def test_the_relay_speaks_whatever_the_server_attaches(monkeypatch, ctrl): + spoken = [] + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: + spoken.append(text) or True) + monkeypatch.setattr(controller_mod.mic, "record_utterance", + lambda *a, **k: np.zeros(10, dtype=np.int16)) + monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's in that folder?") + + def fake_converse(text, on_command=None, on_say=None): + on_say("Let me check that for you.") # what the server now sends + on_command("filectl {\"op\": \"list\", \"path\": \"/tmp\"}") + return Reply("It's got three files in it.") + + monkeypatch.setattr(controller_mod.server_client, "converse", fake_converse) + + ctrl._handle_conversation_turn() + + assert spoken == ["Let me check that for you.", "It's got three files in it."] + + +# ── streamed replies ─────────────────────────────────────────────────────── +# Without streaming the pet waits out the entire model call before saying a +# word. With it, the wait is time-to-first-sentence. + +def test_each_sentence_is_spoken_as_it_arrives(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True) + spoken = [] + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: + spoken.append(text) or True) + + def fake_stream(text, on_say, on_command=None, timeout=180.0): + on_say("The disk is fine.") + on_say("About sixty percent used.") + return controller_mod.server_client.Reply( + "The disk is fine. About sixty percent used.", spoken=True) + + monkeypatch.setattr(controller_mod.server_client, "converse_stream", fake_stream) + monkeypatch.setattr(controller_mod.mic, "record_utterance", + lambda *a, **k: np.zeros(10, dtype=np.int16)) + monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "how's the disk?") + + ctrl._handle_conversation_turn() + + assert spoken == ["The disk is fine.", "About sixty percent used."] + # ...and the empty final reply is not spoken as an extra blank utterance. + assert ctrl.history.entries()[-1].text == "About sixty percent used." + + +def test_a_stream_that_fails_before_speaking_falls_back_silently(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True) + spoken = [] + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: + spoken.append(text) or True) + + def broken_stream(text, on_say, on_command=None, timeout=180.0): + raise controller_mod.server_client.ServerError("no streaming endpoint") + + monkeypatch.setattr(controller_mod.server_client, "converse_stream", broken_stream) + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None, on_say=None: Reply("Fell back fine.")) + + assert ctrl._ask_server("hello").text == "Fell back fine." + assert spoken == [] # nothing was said twice + + +def test_streaming_can_be_switched_off(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False) + called = [] + monkeypatch.setattr(controller_mod.server_client, "converse_stream", + lambda *a, **k: called.append(1)) + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None, on_say=None: Reply("plain path")) + + assert ctrl._ask_server("hi").text == "plain path" + assert called == [] + + +def test_talking_over_a_streamed_reply_still_interrupts(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True) + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False) + + ctrl._speak_stream_chunk("A long explanation you cut short.") + + assert ctrl._talk_now.is_set() + + +# ── notification latency (reported 2026-08-02: 5-10 minutes) ─────────────── +# Draining was wired to the heartbeat's 60s interval, and a heartbeat that +# landed mid-conversation stamped its clock before noticing — so it burned the +# slot and waited another full interval. Several of those in a row is minutes. + +def test_a_notification_goes_out_on_the_next_tick_not_the_next_heartbeat(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "NOTIFICATION_MIN_INTERVAL_SECONDS", 0) + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + sent = [] + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None, on_say=None: + sent.append(text) or Reply("noted")) + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) + monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None) + # A heartbeat has just run, so the next one is a full interval away. + ctrl._last_heartbeat = controller_mod.time.monotonic() + + ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?")) + ctrl._maybe_heartbeat() + + assert sent and "Harry" in sent[0] + + +def test_a_heartbeat_skipped_mid_conversation_does_not_burn_its_slot(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None) + ctrl._last_heartbeat = 0.0 + ctrl._state.transition(PetState.LISTENING) + ctrl._state.transition(PetState.THINKING) + + ctrl._maybe_heartbeat() # due, but the pet is busy + assert ctrl._last_heartbeat == 0.0, "the clock must not advance on a skipped tick" + + ctrl._state.transition(PetState.IDLE) + ctrl._maybe_heartbeat() # free now — runs immediately + assert ctrl._last_heartbeat > 0.0 + + +def test_notifications_wait_while_the_pet_is_mid_turn(monkeypatch, ctrl): + """Speaking over the answer they're already getting would be worse than + waiting a couple of seconds.""" + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + sent = [] + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None, on_say=None: sent.append(text)) + ctrl._state.transition(PetState.LISTENING) + + ctrl._queue_notification(Notification(app="CI", summary="Build finished", body="")) + ctrl._maybe_heartbeat() + + assert sent == [] + assert len(ctrl._pending_notifications) == 1 # kept, not dropped + + +def test_a_second_message_inside_the_rate_limit_is_no_longer_lost(monkeypatch, ctrl): + """The gate used to drop it. With NOTIFICATION_MIN_INTERVAL_SECONDS=60, + two texts a minute apart meant you heard about one of them.""" + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 60) + sent = [] + monkeypatch.setattr(controller_mod.server_client, "converse", + lambda text, on_command=None, on_say=None: + sent.append(text) or Reply("ok")) + monkeypatch.setattr(controller_mod.tts, "speak", + lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True) + + ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?")) + ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="it's urgent")) + ctrl._drain_notifications() + + assert len(sent) == 1 # one round trip... + assert "you there?" in sent[0] and "it's urgent" in sent[0] # ...both messages + assert "2 desktop notifications" in sent[0] + + +def test_the_filter_still_applies(ctrl): + ctrl._notification_gate = controller_mod.notifications.NotificationGate("deploy", 0) + ctrl._queue_notification(Notification(app="Chat", summary="lunch?", body="")) + assert ctrl._pending_notifications == [] + + +def test_a_notification_storm_cannot_become_an_unbounded_backlog(ctrl): + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + for index in range(40): + ctrl._queue_notification(Notification(app="Spam", summary=f"#{index}", body="")) + + assert len(ctrl._pending_notifications) == controller_mod._MAX_PENDING_NOTIFICATIONS + assert "#39" in ctrl._pending_notifications[-1].as_text() # newest kept diff --git a/tests/test_dialogue.py b/tests/test_dialogue.py index 8dadfd7..4745c5f 100644 --- a/tests/test_dialogue.py +++ b/tests/test_dialogue.py @@ -161,7 +161,18 @@ def test_the_relay_report_names_the_cast(): action = dialogue.parse('dialoguectl ' + _scene( '{"voice": "self", "text": "one"}', '{"voice": "narrator", "text": "two"}', )) - assert dialogue.describe(action) == "[dialogue] played 2 lines in 2 voices: narrator, self" + assert dialogue.describe(action).startswith( + "[dialogue] played 2 lines in 2 voices: narrator, self") + + +def test_the_report_says_the_scene_was_already_heard(): + """Observed 2026-07-31: the scene played, then the final reply summarised + it, so the pet said the same thing twice with nothing in between. The + model cannot know the audio already happened unless it is told.""" + action = dialogue.parse('dialoguectl {"lines": [{"text": "hello"}]}') + report = dialogue.describe(action) + assert "HEARD this already" in report + assert "Do not repeat" in report # ── the HTTP request ──────────────────────────────────────────────────────── diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..082d718 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,115 @@ +"""The preflight. Its one job is to never be the thing that's broken. + +A doctor that raises on a broken install diagnoses the wrong patient, so the +tests that matter here are the ugly-input ones: no config at all, a check that +throws, a dependency missing. The individual diagnoses are simple enough to +read; that they *run* on a machine missing everything is the property worth +pinning down. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import config, doctor +from bolt_pet.doctor import FAIL, OK, WARN + + +def test_every_check_returns_a_verdict_on_a_bare_machine(monkeypatch): + """Nothing configured, nothing installed — still a full report.""" + for name in ("SERVER_URL", "API_KEY", "DEEPGRAM_API_KEY", + "ELEVENLABS_API_KEY", "ELEVENLABS_VOICE_ID"): + monkeypatch.setattr(config, name, "") + monkeypatch.setattr(doctor, "_module", lambda _n: False) + + results = doctor.run() + + assert len(results) == len(doctor.CHECKS) + assert all(c.status in (OK, WARN, FAIL) for c in results) + assert all(c.name and c.detail for c in results) + + +def test_a_check_that_raises_does_not_hide_the_others(): + """One broken probe must not cost you the other eleven diagnoses.""" + def explode(*_args): + raise RuntimeError("boom") + + original = doctor.CHECKS + doctor.CHECKS = (("mic", explode),) + original[:2] + try: + results = doctor.run() + finally: + doctor.CHECKS = original + + assert len(results) == 3 + assert results[0].status == FAIL + assert "boom" in results[0].detail + + +def test_missing_server_config_is_a_failure_not_a_warning(monkeypatch): + """Without it the controller exits its thread at startup — the pet looks + alive and simply never answers. That is the worst failure mode there is.""" + monkeypatch.setattr(config, "missing_config", lambda: ["BOLT_SERVER_URL", "DESK_API_KEY"]) + check = doctor.check_config() + assert check.status == FAIL + assert "BOLT_SERVER_URL" in check.detail + assert check.fix + + +def test_a_configured_server_passes_without_being_contacted(monkeypatch): + """The shallow run must not need the network — it's the first thing you + reach for when the network is what's wrong.""" + monkeypatch.setattr(config, "missing_config", lambda: []) + monkeypatch.setattr(config, "SERVER_URL", "http://bolt.local:8000") + assert doctor.check_config().status == OK + assert doctor.check_server(deep=False).status == OK + + +def test_every_problem_comes_with_something_to_do_about_it(monkeypatch): + """"screen reading: warn" is useless on its own; "apt install tesseract-ocr" + is the entire point of the tool.""" + for name in ("SERVER_URL", "API_KEY", "DEEPGRAM_API_KEY"): + monkeypatch.setattr(config, name, "") + monkeypatch.setattr(doctor, "_module", lambda _n: False) + + for check in doctor.run(): + if check.status == FAIL: + assert check.fix, f"{check.name} says what's wrong but not what to do" + + +def test_the_exit_code_is_nonzero_only_for_real_failures(monkeypatch, capsys): + monkeypatch.setattr(doctor, "run", lambda deep=False: [ + doctor.Check("a", OK, "fine"), doctor.Check("b", WARN, "degraded")]) + assert doctor.main([]) == 0 + + monkeypatch.setattr(doctor, "run", lambda deep=False: [doctor.Check("a", FAIL, "broken")]) + assert doctor.main([]) == 1 + assert "Fix those first" in capsys.readouterr().out + + +def test_deep_is_off_unless_asked(monkeypatch): + seen = [] + monkeypatch.setattr(doctor, "run", lambda deep=False: seen.append(deep) or []) + doctor.main([]) + doctor.main(["--deep"]) + assert seen == [False, True] + + +def test_a_slow_silence_timeout_is_flagged(monkeypatch): + """The setting most likely to make it feel sluggish, and the least obvious + — it is pure dead air before anything at all starts happening.""" + monkeypatch.setattr(config, "SILENCE_END_SEC", 2.0) + check = doctor.check_latency() + assert check.status == WARN + assert "2s" in check.detail or "2 " in check.detail + + monkeypatch.setattr(config, "SILENCE_END_SEC", 0.9) + assert doctor.check_latency().status == OK + + +def test_a_check_line_renders_the_fix_only_when_there_is_a_problem(): + assert "→" not in doctor.Check("x", OK, "all good", fix="unused").line() + assert "→ do the thing" in doctor.Check("x", WARN, "hmm", fix="do the thing").line() diff --git a/tests/test_lipsync.py b/tests/test_lipsync.py new file mode 100644 index 0000000..7451fb8 --- /dev/null +++ b/tests/test_lipsync.py @@ -0,0 +1,135 @@ +"""Amplitude → mouth openness, from PCM samples to the drawn frame. + +Two halves, tested separately because only one of them needs a display: +`tts.level_of`/`envelope` turn samples into a 0..1 loudness, and +`PetWindow._mouth_frame` turns that loudness into a frame index. They meet at +the `mouth` signal, which the pipeline smoke test covers end to end. +""" + +import sys +import time +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet.audio import tts + + +def frame(amplitude: int, samples: int = 480) -> np.ndarray: + return np.full(samples, amplitude, dtype=np.int16) + + +# ── loudness ──────────────────────────────────────────────────────────────── + +def test_silence_closes_the_mouth(): + assert tts.level_of(np.zeros(480, dtype=np.int16)) == 0.0 + + +def test_a_loud_frame_opens_it_fully(): + assert tts.level_of(frame(30000)) == 1.0 + + +def test_the_level_never_leaves_zero_to_one(): + """It indexes a frame list; out of range is an IndexError on the UI thread.""" + for amplitude in (0, 1, 500, 6000, 20000, 32767): + assert 0.0 <= tts.level_of(frame(amplitude)) <= 1.0 + + +def test_it_rises_with_amplitude(): + quiet, middling, loud = (tts.level_of(frame(a)) for a in (800, 4000, 12000)) + assert quiet < middling < loud + + +def test_quiet_speech_still_moves_the_mouth_visibly(): + """The sqrt curve is the whole point. Speech spends most of its time well + below peak, so a linear map leaves the mouth barely open for normal talking + and the pet looks like it's mumbling.""" + assert tts.level_of(frame(1500)) > 0.15 + + +def test_an_empty_frame_is_silence_not_a_crash(): + assert tts.level_of(np.zeros(0, dtype=np.int16)) == 0.0 + + +# ── the envelope of a whole clip ──────────────────────────────────────────── + +def test_an_envelope_has_one_value_per_frame_at_the_requested_rate(): + one_second = np.zeros(16000, dtype=np.int16) + assert len(tts.envelope(one_second, 16000, fps=30)) == pytest.approx(30, abs=1) + + +def test_an_envelope_tracks_loud_and_quiet_stretches(): + pcm = np.concatenate([np.zeros(8000, dtype=np.int16), frame(20000, 8000)]) + levels = tts.envelope(pcm, 16000, fps=10) + assert max(levels[:4]) == 0.0 # the silent half + assert min(levels[-4:]) > 0.5 # the loud half + + +def test_an_empty_clip_has_an_empty_envelope(): + assert tts.envelope(np.zeros(0, dtype=np.int16), 16000) == [] + + +# ── the drawn frame ───────────────────────────────────────────────────────── + +@pytest.fixture +def window(): + """Needs a QApplication; run with QT_QPA_PLATFORM=offscreen.""" + from PySide6.QtWidgets import QApplication + + from bolt_pet.ui.pet_window import PetWindow + + _app = QApplication.instance() or QApplication(["test"]) + win = PetWindow() + yield win + win.close() + + +class FakeAnimation: + """Stands in for sprite.Animation — _mouth_frame only wants `.frames`.""" + + def __init__(self, frames): + self.frames = list(frames) + + +def test_the_talking_frame_follows_the_level(window): + """The talking frames are an openness ramp — closed first, widest last — + specifically so loudness can index them directly.""" + animation = FakeAnimation(["closed", "a", "b", "c", "d", "open"]) + + assert window._mouth_frame(animation) is None # no level set yet + + window.set_mouth(0.0) + assert window._mouth_frame(animation) == "closed" + + window.set_mouth(1.0) + assert window._mouth_frame(animation) == "open" + + window.set_mouth(0.5) + assert window._mouth_frame(animation) not in ("closed", "open") + + +def test_a_stale_level_gives_the_animation_back(window): + """If the audio thread stops sending levels — TTS died, the clip ended + without a final zero — the mouth must not freeze half-open forever. After a + moment it falls back to the ordinary looping animation.""" + animation = FakeAnimation(["a", "b", "c"]) + window.set_mouth(0.9) + assert window._mouth_frame(animation) is not None + + window._mouth_at = time.monotonic() - 5.0 + assert window._mouth_frame(animation) is None + + +def test_a_single_frame_animation_falls_back_instead_of_indexing(window): + """Art with one talking frame can't lip-sync; it must not try.""" + window.set_mouth(1.0) + assert window._mouth_frame(FakeAnimation(["only"])) is None + assert window._mouth_frame(FakeAnimation([])) is None + + +def test_no_animation_at_all_is_handled(window): + window.set_mouth(0.5) + assert window._mouth_frame(None) is None diff --git a/tests/test_pipeline_smoke.py b/tests/test_pipeline_smoke.py new file mode 100644 index 0000000..2c7bd6f --- /dev/null +++ b/tests/test_pipeline_smoke.py @@ -0,0 +1,289 @@ +"""End-to-end: microphone in, speech out, over real HTTP. + +Every other test in this suite injects a fake at the seam it cares about, and +every one of them passed all week while these got through to production: + + - notifications sitting unspoken for minutes (a clock stamped in the wrong + order, two correct units) + - the pet saying the same thing twice (server discarded prose, model repeated + it — both sides behaving as written) + - `[laughing]` read out loud (a tag that means something to one model and + nothing to the next one down the pipe) + - a device command written as prose (extractor fine, prompt fine, no marker) + +They were all *interaction* bugs. So this one runs the actual pipeline against +a real socket: a threaded HTTP server that speaks the desk protocol, the real +`server_client` doing real requests (including NDJSON streaming), the real +controller loop and state machine. The only fakes are where the hardware is — +the mic stream and the speakers — because those are the two things a test +genuinely cannot have. +""" + +import json +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import numpy as np +import pytest +from PySide6.QtWidgets import QApplication + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import controller as controller_mod +from bolt_pet.notifications import Notification +from bolt_pet.state import PetState + +_app = QApplication.instance() or QApplication(["test"]) + + +# ── a desk server that actually listens on a port ─────────────────────────── + +class FakeDesk: + """Scripted responses, real HTTP. Set `.script` per test.""" + + def __init__(self): + self.script = {} + self.requests = [] + self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler()) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + @property + def url(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + def stop(self) -> None: + self._server.shutdown() + self._server.server_close() + + def _handler(self): + desk = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass # the test output is not an access log + + def do_GET(self): + path = self.path.split("?")[0] + desk.requests.append(("GET", path)) + self._json(desk.script.get(path, {"files": []})) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + path = self.path.split("?")[0] + desk.requests.append(("POST", path, body)) + response = desk.script.get(path) + if callable(response): + response = response(body) + if path.endswith("converse_stream"): + self._ndjson(response or []) + else: + self._json(response if response is not None else {"type": "reply", "text": "ok"}) + + def _json(self, payload): + data = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _ndjson(self, events): + self.send_response(200) + self.send_header("Content-Type", "application/x-ndjson") + self.end_headers() + for event in events: + self.wfile.write((json.dumps(event) + "\n").encode()) + self.wfile.flush() + + return Handler + + +class FakeMic: + """Loud frames then quiet ones, so the VAD ends the utterance on its own.""" + + def __init__(self, loud=8, quiet=60): + self.frames = ([np.full((320, 1), 4000, dtype=np.int16)] * loud + + [np.zeros((320, 1), dtype=np.int16)] * quiet) + + def read(self, _n): + return (self.frames.pop(0) if self.frames + else np.zeros((320, 1), dtype=np.int16)), None + + def __enter__(self): + return self + + def __exit__(self, *_a): + return False + + +@pytest.fixture +def pipeline(monkeypatch): + """A controller wired to a real local server, with fake ears and mouth.""" + desk = FakeDesk() + spoken: list[str] = [] + levels: list[float] = [] + + monkeypatch.setattr(controller_mod.config, "SERVER_URL", desk.url) + monkeypatch.setattr(controller_mod.config, "API_KEY", "test-key") + monkeypatch.setattr(controller_mod.config, "SESSION_ID", "pet-smoke") + monkeypatch.setattr(controller_mod.config, "SILENCE_END_SEC", 0.2) + monkeypatch.setattr(controller_mod.config, "MIN_UTTERANCE_S", 0.0) + monkeypatch.setattr(controller_mod.config, "RECEIVE_FILES", False) + # Off by default so each test picks its own path; the streaming tests opt in. + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False) + monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text) + monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False) + # Deepgram and the speakers are the two things a test can't have. + monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's on my disk?") + monkeypatch.setattr(controller_mod.stt_stream.StreamingTranscriber, "open", + classmethod(lambda cls, **kw: None)) + + def fake_speak(text, on_error=None, should_stop=None, voice_id=None, on_level=None): + spoken.append(text) + if on_level is not None: + on_level(0.8) # the mouth opens while a word plays... + levels.append(0.8) + return True + + monkeypatch.setattr(controller_mod.tts, "speak", fake_speak) + + ctrl = controller_mod.PetController() + ctrl._stream = FakeMic() + try: + yield ctrl, desk, spoken, levels + finally: + desk.stop() + + +# ── the whole path ────────────────────────────────────────────────────────── + +def test_a_plain_turn_goes_mic_to_speaker(pipeline): + ctrl, desk, spoken, _levels = pipeline + desk.script["/desk/converse"] = {"type": "reply", "text": "About sixty percent full."} + + ctrl._handle_conversation_turn() + + assert spoken == ["About sixty percent full."] + assert ctrl._state.state == PetState.IDLE + posted = [r for r in desk.requests if r[0] == "POST"] + assert posted[0][1] == "/desk/converse" + assert "what's on my disk?" in posted[0][2]["text"] + assert ctrl.history.entries()[-1].text == "About sixty percent full." + + +def test_a_streamed_turn_speaks_each_sentence_as_it_lands(pipeline, monkeypatch): + """The NDJSON is parsed by the real client over a real socket — the layer + that a mocked `converse` can never exercise.""" + ctrl, desk, spoken, _levels = pipeline + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True) + desk.script["/desk/converse_stream"] = [ + {"type": "say", "text": "The disk is fine."}, + {"type": "say", "text": "About sixty percent used."}, + {"type": "reply", "text": "The disk is fine. About sixty percent used.", + "already_spoken": True}, + ] + + ctrl._handle_conversation_turn() + + assert spoken == ["The disk is fine.", "About sixty percent used."] + # The final reply must not be spoken a third time. + assert len(spoken) == 2 + + +def test_a_tool_turn_says_give_me_a_sec_then_the_answer(pipeline, monkeypatch): + """Holding line, relayed command, and the real answer — in that order.""" + ctrl, desk, spoken, _levels = pipeline + ran = [] + monkeypatch.setattr(controller_mod.server_client, "run_local_command", + lambda cmd: ran.append(cmd) or "[exit 0]\n60% used") + desk.script["/desk/converse"] = { + "type": "command", "command": "df -h /", "token": "tok", + "say": "Let me check that for you.", + } + desk.script["/desk/tool_result"] = {"type": "reply", "text": "Sixty percent used."} + + ctrl._handle_conversation_turn() + + assert spoken == ["Let me check that for you.", "Sixty percent used."] + assert ran == ["df -h /"] + relayed = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/tool_result") + assert relayed[2]["output"].endswith("60% used") + + +def test_a_device_command_never_reaches_the_shell(pipeline, monkeypatch): + ctrl, desk, spoken, _levels = pipeline + ran = [] + monkeypatch.setattr(controller_mod.server_client, "run_local_command", lambda cmd: ran.append(cmd)) + actions = [] + ctrl.action.connect(actions.append) + desk.script["/desk/converse"] = { + "type": "command", "command": "petctl emote wave", "token": "tok"} + desk.script["/desk/tool_result"] = {"type": "reply", "text": "There you go."} + + ctrl._handle_conversation_turn() + + assert ran == [] + assert actions == [{"action": "emote", "emote": "wave"}] + assert spoken == ["There you go."] + + +def test_the_mouth_moves_with_the_audio(pipeline): + """Lip-sync is only real if the level actually reaches the window.""" + ctrl, desk, _spoken, levels = pipeline + mouth = [] + ctrl.mouth.connect(mouth.append) + desk.script["/desk/converse"] = {"type": "reply", "text": "Talking now."} + + ctrl._handle_conversation_turn() + + assert levels, "TTS was never given a level callback" + assert 0.8 in mouth # opened while speaking... + assert mouth[-1] == 0.0 # ...and closed at the end + + +def test_a_notification_is_forwarded_and_spoken(pipeline): + """The path that was silently sitting for five to ten minutes.""" + ctrl, desk, spoken, _levels = pipeline + ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0) + desk.script["/desk/converse"] = {"type": "reply", "text": "Harry says he's around."} + + ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?")) + ctrl._maybe_heartbeat() + + assert spoken == ["Harry says he's around."] + forwarded = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/converse") + assert "Harry" in forwarded[2]["text"] + + +def test_streaming_falls_back_when_the_server_is_older(pipeline, monkeypatch): + """A server without /desk/converse_stream (or one that answers with nothing) + must not cost a turn — the client drops to the plain endpoint. This is how + the pet keeps working against a container that hasn't been updated yet.""" + ctrl, desk, spoken, _levels = pipeline + monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True) + desk.script["/desk/converse_stream"] = [] # nothing streamed back + desk.script["/desk/converse"] = {"type": "reply", "text": "Still here."} + + ctrl._handle_conversation_turn() + + assert spoken == ["Still here."] + paths = [r[1] for r in desk.requests if r[0] == "POST"] + assert paths == ["/desk/converse_stream", "/desk/converse"] + + +def test_a_dead_server_leaves_the_pet_usable(pipeline): + """It should flash an error and go back to listening, not wedge.""" + ctrl, desk, spoken, _levels = pipeline + desk.stop() # the server disappears mid-session + states = [] + ctrl.state_changed.connect(states.append) + + ctrl._handle_conversation_turn() + + assert states[-2:] == ["error", "idle"] + assert spoken == [] diff --git a/tests/test_speech.py b/tests/test_speech.py new file mode 100644 index 0000000..00b39db --- /dev/null +++ b/tests/test_speech.py @@ -0,0 +1,260 @@ +"""The rules about talking — the four kinds of utterance and the mic policy. + +These were four near-copies in the controller before, and the copies had +drifted: one didn't arm barge-in, one skipped the follow-up rule. The value of +having one `Speaker` is only real if the differences between the kinds stay +*visible*, so this asserts on the differences rather than on the machinery. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import config, speech +from bolt_pet.speech import Speaker, Utterance +from bolt_pet.state import PetState, PetStateMachine + + +class FakeTts: + def __init__(self, completed=True): + self.completed = completed + self.calls = [] + + def speak(self, text, on_error=None, should_stop=None, voice_id=None, on_level=None): + self.calls.append({"text": text, "voice_id": voice_id, + "interruptible": should_stop is not None}) + if on_level is not None: + on_level(0.7) + return self.completed + + def play_pcm(self, pcm, sample_rate, should_stop=None, on_level=None): + self.calls.append({"pcm": pcm, "rate": sample_rate, + "interruptible": should_stop is not None}) + return self.completed + + +class FakeBargeIn: + def __init__(self): + self.resets = 0 + + def reset(self): + self.resets += 1 + + def check(self, _frame=None): + return False + + +def build(**kwargs): + """A speaker plus the things worth asserting on.""" + state = PetStateMachine() + tts = kwargs.pop("tts", None) or FakeTts() + said, logged, recorded, levels = [], [], [], [] + speaker = Speaker( + state=state, tts=tts, + history=recorded.append, + on_said=said.append, + on_log=logged.append, + on_level=levels.append, + **kwargs, + ) + return speaker, state, tts, said, logged, recorded, levels + + +# ── what distinguishes the four kinds ─────────────────────────────────────── + +def test_a_reply_is_recorded_but_a_holding_line_is_not(): + """Filler must not push the actual answer out of the transcript.""" + speaker, state, _tts, _said, _logged, recorded, _levels = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.holding("Give me a sec.")) + speaker.say(Utterance.reply("Sixty percent used.")) + + assert recorded == ["Sixty percent used."] + + +def test_a_holding_line_resumes_the_turn_it_interrupted(): + """The turn isn't over — a tool is still running — so it must go back to + THINKING, not drop to IDLE and end the turn.""" + speaker, state, _tts, *_ = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.holding("Let me check.")) + + assert state.state == PetState.THINKING + + +def test_a_holding_line_cannot_be_talked_over(): + """Cutting off "give me a sec" strands the tool that's already running.""" + detector = FakeBargeIn() + speaker, state, tts, *_ = build(barge_in=lambda: detector) + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.holding("One sec.")) + assert tts.calls[-1]["interruptible"] is False + + speaker.say(Utterance.reply("Done.")) + assert tts.calls[-1]["interruptible"] is True + + +def test_a_streamed_sentence_stays_talking_between_sentences(): + """Otherwise the sprite flickers idle-talking-idle down a long answer.""" + speaker, state, *_ = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.stream_chunk("The disk is fine.")) + + assert state.state == PetState.TALKING + + +def test_a_scene_is_recorded_because_the_user_heard_it(): + speaker, state, _tts, _said, _logged, recorded, _levels = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say_pcm(Utterance.scene("Once upon a time."), pcm=b"\x00\x00", sample_rate=44100) + + assert recorded == ["Once upon a time."] + assert state.state == PetState.THINKING + + +# ── barge-in ordering, which is what the copies got wrong ─────────────────── + +def test_the_detector_is_reset_after_playback_not_only_before(): + """Playback fed the pet's own voice into the wake model's window. If it + isn't cleared afterwards, the idle listener re-hears the last sentence and + the pet answers itself.""" + detector = FakeBargeIn() + speaker, state, *_ = build(barge_in=lambda: detector) + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("Hello there.")) + + assert detector.resets == 2 # armed before, cleared after + + +def test_the_barge_in_detail_is_captured_before_the_reset(): + """Read it after the reset and every interruption reports zeroed counters — + which reads like hard evidence and is nothing of the sort.""" + detector = FakeBargeIn() + details = ["score 0.81 at frame 12", "score 0.000 at frame 0"] + speaker, state, *_ = build(barge_in=lambda: detector, + detail_of=lambda: details.pop(0)) + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("Hello there.")) + + assert speaker.last_detail == "score 0.81 at frame 12" + + +def test_a_detector_that_appears_late_is_still_used(): + """The detector is built after the speaker — it needs the mic stream — so + it's read through a callable. Holding a copy is how the two drift apart.""" + detector = None + speaker, state, tts, *_ = build(barge_in=lambda: detector) + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("Before.")) + assert tts.calls[-1]["interruptible"] is False + + detector = FakeBargeIn() + speaker.say(Utterance.reply("After.")) + assert tts.calls[-1]["interruptible"] is True + + +# ── the mouth ─────────────────────────────────────────────────────────────── + +def test_the_mouth_is_closed_when_the_line_ends(): + """A pet left mid-vowel after the audio stops looks broken.""" + speaker, state, _tts, _said, _logged, _recorded, levels = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("Talking.")) + + assert levels[0] > 0 + assert levels[-1] == 0.0 + + +# ── text handling ─────────────────────────────────────────────────────────── + +def test_an_empty_line_is_not_spoken_at_all(): + """A reply that is nothing but an audio tag reduces to '' — and a silent + bubble with no audio is better than the pet announcing "laughing".""" + speaker, state, tts, said, *_ = build() + + assert speaker.say(Utterance.reply("[laughing]")) is True + assert tts.calls == [] + assert said == [] + assert state.state == PetState.IDLE + + +def test_the_bubble_gets_display_text_and_tts_gets_the_original(): + """The bubble keeps emoji and drops markdown; TTS does its own stripping + (inside speak(), so every path is covered) and needs the real text.""" + speaker, state, tts, said, *_ = build() + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("**Sixty** percent 🎉")) + + assert said == ["Sixty percent 🎉"] + assert tts.calls[-1]["text"] == "**Sixty** percent 🎉" + + +def test_a_voice_override_reaches_tts(): + speaker, state, tts, *_ = build(voice_id=lambda: "voice-abc") + state.transition(PetState.LISTENING) + state.transition(PetState.THINKING) + + speaker.say(Utterance.reply("In character.")) + + assert tts.calls[-1]["voice_id"] == "voice-abc" + + +# ── the follow-up rule ────────────────────────────────────────────────────── + +@pytest.fixture +def follow_ups_on(monkeypatch): + monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", True) + monkeypatch.setattr(config, "FOLLOW_UP_MAX_TURNS", 3) + + +def test_an_interruption_always_reopens_the_mic(follow_ups_on): + """You talked over it — you are mid-sentence, so it has to listen.""" + keep, why = speech.follow_up_decision("Anything else?", completed=False, follow_ups=99) + assert keep is True + assert why == "interrupted" + + +def test_a_question_keeps_the_mic_open_without_the_wake_word(follow_ups_on): + keep, why = speech.follow_up_decision("Want me to check?", completed=True, follow_ups=0) + assert (keep, why) == (True, "question") + + +def test_a_statement_ends_the_turn(follow_ups_on): + keep, _why = speech.follow_up_decision("Sixty percent used.", completed=True, follow_ups=0) + assert keep is False + + +def test_the_cap_stops_a_server_that_ends_every_reply_with_a_question(follow_ups_on): + """Otherwise mic noise loops it forever.""" + assert speech.follow_up_decision("Ok?", completed=True, follow_ups=2)[0] is True + keep, why = speech.follow_up_decision("Ok?", completed=True, follow_ups=3) + assert keep is False + assert "cap" in why + + +def test_the_rule_can_be_switched_off(monkeypatch): + monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", False) + assert speech.follow_up_decision("Ok?", completed=True, follow_ups=0)[0] is False diff --git a/tests/test_speech_text.py b/tests/test_speech_text.py index 53c9d5f..38dc385 100644 --- a/tests/test_speech_text.py +++ b/tests/test_speech_text.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from bolt_pet.speech_text import for_display, for_speech, is_question @@ -77,3 +79,33 @@ def test_is_question_ignores_question_marks_that_are_not_spoken(): assert not is_question("Docs are at https://example.com/x?y=1") assert not is_question("") assert not is_question(None) + + +# ── ElevenLabs v3 delivery tags (observed live 2026-07-31) ────────────────── +# "[laughing] That one came through clean" was spoken as "laughing That one +# came through clean": the tags mean something to the dialogue model inside a +# dialoguectl scene, and nothing at all to the ordinary reply voice. + +def test_delivery_tags_are_never_spoken_aloud(): + spoken = for_speech("[laughing] That one came through clean.") + assert "laughing" not in spoken + assert spoken.startswith("That one came through clean") + + +def test_the_bubble_does_not_caption_a_laugh_nobody_heard(): + assert "[laughing]" not in for_display("[laughing] All good.") + + +@pytest.mark.parametrize("tag", [ + "[whispering]", "[cheerfully]", "[sighs]", "[laughs]", "[clears throat]", + "[nervously]", "[excited]", "[pause]", "[shouting]", +]) +def test_the_common_tags_are_all_covered(tag): + assert "" == for_speech(tag).strip() + + +def test_ordinary_bracketed_text_survives(): + """The tags are matched narrowly on purpose — real bracketed content is + part of what the user asked to hear.""" + assert "1" in for_speech("See reference [1] for details.") + assert "docs" in for_speech("It's in [the docs] somewhere.") diff --git a/tests/test_stt_stream.py b/tests/test_stt_stream.py new file mode 100644 index 0000000..5c85e14 --- /dev/null +++ b/tests/test_stt_stream.py @@ -0,0 +1,173 @@ +"""Streaming speech-to-text: the protocol, and the fallback that makes it safe +to switch on at all. + +A fake websocket throughout — no network, no Deepgram account, no audio. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet.audio import mic +from bolt_pet.audio.stt_stream import StreamingTranscriber + + +class FakeSocket: + """Records what was sent; replays scripted Deepgram frames.""" + + def __init__(self, messages=(), fail_on_send=False): + self.sent = [] + self.closed = False + self.fail_on_send = fail_on_send + self._messages = list(messages) + + def send_binary(self, data): + if self.fail_on_send: + raise ConnectionError("socket died") + self.sent.append(data) + + def send(self, text): + self.sent.append(text) + + def recv(self): + if self._messages: + return self._messages.pop(0) + time.sleep(0.01) + raise ConnectionError("closed") + + def close(self): + self.closed = True + + +def _results(transcript, is_final=True): + return json.dumps({ + "type": "Results", "is_final": is_final, + "channel": {"alternatives": [{"transcript": transcript}]}, + }) + + +def _frame(value=1000): + return np.full(320, value, dtype=np.int16) + + +# ── the protocol ──────────────────────────────────────────────────────────── + +def test_frames_go_up_as_they_are_captured(): + socket = FakeSocket([_results("what's the weather")]) + session = StreamingTranscriber(socket) + + for _ in range(3): + session.feed(_frame()) + text = session.finish() + + assert len(socket.sent) == 4 # three frames plus the close message + assert text == "what's the weather" + assert socket.closed + + +def test_only_final_results_are_kept(): + """Interim hypotheses change under you; concatenating them would produce + "what what's what's the what's the weather".""" + socket = FakeSocket([ + _results("what's", is_final=False), + _results("what's the", is_final=False), + _results("what's the weather", is_final=True), + ]) + session = StreamingTranscriber(socket) + assert session.finish() == "what's the weather" + + +def test_several_final_segments_are_joined(): + socket = FakeSocket([_results("turn the lights on"), _results("in the kitchen")]) + session = StreamingTranscriber(socket) + assert session.finish() == "turn the lights on in the kitchen" + + +def test_junk_frames_are_ignored_rather_than_killing_the_reader(): + """An exception on the reader thread would silently end transcription for + the rest of the utterance.""" + socket = FakeSocket(["not json at all", '{"type":"Metadata"}', + _results("still works")]) + session = StreamingTranscriber(socket) + assert session.finish() == "still works" + + +def test_an_empty_frame_means_the_socket_closed(): + """websocket-client returns "" from recv() on a closed connection, so it + ends the read loop rather than being treated as a blank transcript.""" + socket = FakeSocket([_results("heard this much"), "", _results("never arrives")]) + session = StreamingTranscriber(socket) + assert session.finish() == "heard this much" + + +def test_a_socket_that_dies_mid_utterance_gives_up_quietly(): + socket = FakeSocket([], fail_on_send=True) + session = StreamingTranscriber(socket) + + session.feed(_frame()) # must not raise — recording carries on + assert session.finish() == "" + + +# ── opening: failure is an ordinary outcome ──────────────────────────────── + +def test_open_returns_none_when_it_cannot_connect(): + """None means "the one-shot path will do it", not an error.""" + def refuse(): + raise OSError("no network") + + assert StreamingTranscriber.open(connect=refuse) is None + + +def test_open_returns_a_session_when_it_can(): + session = StreamingTranscriber.open(connect=lambda: FakeSocket([_results("hi")])) + assert session is not None + assert session.finish() == "hi" + + +def test_streaming_is_off_without_the_switch_or_a_key(monkeypatch): + from bolt_pet.audio import stt_stream + + monkeypatch.setattr(stt_stream.config, "STT_STREAMING", False) + assert stt_stream.available() is False + monkeypatch.setattr(stt_stream.config, "STT_STREAMING", True) + monkeypatch.setattr(stt_stream.config, "DEEPGRAM_API_KEY", "") + assert stt_stream.available() is False + + +# ── the capture hook ──────────────────────────────────────────────────────── + +class _Stream: + """Loud frames, then quiet ones, so the VAD ends the utterance.""" + + def __init__(self, loud=6, quiet=40): + self.frames = ([np.full((320, 1), 3000, dtype=np.int16)] * loud + + [np.zeros((320, 1), dtype=np.int16)] * quiet) + + def read(self, n): + return (self.frames.pop(0) if self.frames + else np.zeros((320, 1), dtype=np.int16)), None + + +def test_recording_hands_every_speech_frame_to_the_listener(): + seen = [] + pcm = mic.record_utterance(_Stream(), on_frame=seen.append, + silence_end_sec=0.2, min_utterance_s=0.0) + assert pcm is not None + assert len(seen) >= 6 # every frame of speech was streamed + + +def test_a_listener_that_throws_cannot_break_the_recording(): + """The fallback is about to need this audio — a dead stream must not cost + the recording too.""" + def explode(frame): + raise RuntimeError("stream died") + + pcm = mic.record_utterance(_Stream(), on_frame=explode, + silence_end_sec=0.2, min_utterance_s=0.0) + assert pcm is not None and len(pcm) > 0 diff --git a/tests/test_window_state.py b/tests/test_window_state.py new file mode 100644 index 0000000..03b8950 --- /dev/null +++ b/tests/test_window_state.py @@ -0,0 +1,84 @@ +"""Remembering where the pet was left — and refusing to when that's a trap.""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import window_state + + +def test_a_saved_position_comes_back(tmp_path): + path = tmp_path / "window.json" + window_state.save(1200, 640, path) + assert window_state.load(path) == (1200, 640) + + +def test_no_file_yet_is_not_an_error(tmp_path): + assert window_state.load(tmp_path / "nope.json") is None + + +def test_a_corrupt_file_falls_back_to_the_default_corner(tmp_path): + """Whatever is in there, the pet still has to start.""" + path = tmp_path / "window.json" + for junk in ("", "{", "null", "[]", '{"x": "left"}', '{"y": 3}'): + path.write_text(junk) + assert window_state.load(path) is None + + +def test_saving_creates_the_cache_directory(tmp_path): + path = tmp_path / "deep" / "nested" / "window.json" + window_state.save(10, 20, path) + assert window_state.load(path) == (10, 20) + + +def test_an_unwritable_location_is_swallowed(tmp_path): + """A read-only cache dir is a reason to forget the position, not to crash + on every drag.""" + window_state.save(1, 2, tmp_path / "no" / "\0bad" / "window.json") + + +def test_the_write_is_atomic_and_leaves_no_litter(tmp_path): + path = tmp_path / "window.json" + window_state.save(5, 5, path) + window_state.save(7, 7, path) + assert json.loads(path.read_text()) == {"x": 7, "y": 7} + assert [p.name for p in tmp_path.iterdir()] == ["window.json"] + + +# ── validating against the screens that exist *now* ───────────────────────── + +LAPTOP = (0, 0, 1920, 1080) +EXTERNAL = (1920, 0, 4480, 1440) + + +def test_a_position_on_a_connected_screen_is_kept(): + assert window_state.is_visible_on(1700, 900, 128, [LAPTOP]) + + +def test_a_position_on_an_unplugged_monitor_is_refused(): + """The dangerous case: restoring it faithfully puts the pet somewhere you + can't see or reach.""" + assert not window_state.is_visible_on(3000, 700, 128, [LAPTOP]) + assert window_state.is_visible_on(3000, 700, 128, [LAPTOP, EXTERNAL]) + + +def test_mostly_off_screen_counts_as_gone(): + """A few pixels of ear poking onto the desktop is not "reachable".""" + assert not window_state.is_visible_on(1910, 500, 128, [LAPTOP]) + assert window_state.is_visible_on(1830, 500, 128, [LAPTOP]) + + +def test_touching_an_edge_is_not_overlapping(): + assert not window_state.is_visible_on(1920, 0, 128, [LAPTOP]) + + +def test_negative_coordinates_are_fine_when_a_screen_is_there(): + """Monitors left of or above the primary have negative origins.""" + left_of_primary = (-1920, 0, 0, 1080) + assert window_state.is_visible_on(-900, 400, 128, [left_of_primary, LAPTOP]) + + +def test_no_screens_at_all_is_not_visible(): + assert not window_state.is_visible_on(100, 100, 128, [])