Add example environment configuration file for setup

- Implement `list_input_devices` to retrieve and filter input-capable audio devices.
- Create `MicPickerDialog` for selecting and saving microphone settings.
- Add `set_env_value` function to update .env file with selected microphone index.
- Update UI components to integrate microphone selection functionality.
- Enhance tests for microphone device handling and environment variable management.

.env.example

@@ -5,7 +5,7 @@
 # DESK_API_KEY is the master key from the server's own .env, or a per-user
 # key minted via the api_key_generate desk marker (see the main repo's
 # CLAUDE.md "Per-user API keys" section).
-BOLT_SERVER_URL=http://your-server:5002
+BOLT_SERVER_URL=https://api.themajesticnetwork.com
 DESK_API_KEY=
 
 # Unique per machine/install so sessions don't collide. Defaults to
@@ -17,19 +17,18 @@ DESK_API_KEY=
 # the project root next to this file. Point WAKE_MODEL_FILE at a different
 # .onnx model to change the phrase (same convention as desk_client's
 # WAKE_MODEL_FILE / bolt.onnx in the main repo).
-#WAKE_MODEL_FILE=thunderbolt.onnx
+WAKE_MODEL_FILE=thunderbolt.onnx
-#WAKE_WORD_THRESHOLD=0.5
+WAKE_WORD_THRESHOLD=0.6
-#WAKE_CHECK_INTERVAL_SECONDS=1.2
+WAKE_CHECK_INTERVAL_SECONDS=1.2
 
-# ── STT (Deepgram) ──────────────────────────────────────────────────────────
+# ── STT / TTS (server-hosted — same /desk/stt and /desk/tts the Android app
-DEEPGRAM_API_KEY=
+# uses) ──────────────────────────────────────────────────────────────────────
-#DEEPGRAM_MODEL=nova-3
+# No Deepgram or ElevenLabs account needed here: both go through
-
+# BOLT_SERVER_URL/DESK_API_KEY above, the same as the rest of this file.
-# ── TTS (ElevenLabs) — omit to use offline TTS only ─────────────────────────
+# ELEVENLABS_VOICE_ID just tells the server which voice to request — omit it
-ELEVENLABS_API_KEY=
+# to fall back to offline pyttsx3 TTS instead (STT still works either way).
-ELEVENLABS_VOICE_ID=
+ELEVENLABS_VOICE_ID=UszkcUdO0dD9JrXX8DDE
-#ELEVENLABS_MODEL_ID=eleven_flash_v2
+#TTS_SAMPLE_RATE=16000
-#TTS_SAMPLE_RATE=24000
 
 # Ask Bolt to use a different voice (or another language) and the server
 # picks one from the ElevenLabs voice library and tags the reply with it.
@@ -40,10 +39,14 @@ ELEVENLABS_VOICE_ID=
 #VOICE_STICKY=true
 
 # ── Multi-voice dialogue (ElevenLabs Text to Dialogue) ──────────────────────
+# The one feature the server has no endpoint for, so this is the only place
+# in the whole app that still needs a local ElevenLabs API key — everything
+# else (the normal reply voice, transcription) goes through the server above.
 # Lets Bolt play a short scene in several voices with delivery tags the v3
 # model acts on ("[cheerfully] Hello", "[whispering] He is lying"), driven by
 # the server through a relayed `dialoguectl` command. Name the cast here —
 # "self" always means whatever voice the pet is currently using.
+ELEVENLABS_API_KEY=
 #DIALOGUE=true
 #DIALOGUE_MODEL_ID=eleven_v3
 #DIALOGUE_VOICES=narrator:9BWtsMINqrJLrRacOk9x,villain:IKne3meq5aSn9XLyUdCD
@@ -57,12 +60,11 @@ ELEVENLABS_VOICE_ID=
 #SELF_RESTART=true
 #SELF_RESTART_MAX=5
 #SELF_RESTART_WINDOW_SECONDS=900
-# Used instead of ELEVENLABS_MODEL_ID whenever the server picked the voice
-# or the reply has non-ASCII in it — the flash_v2 default is English-only.
-#ELEVENLABS_MULTILINGUAL_MODEL_ID=eleven_flash_v2_5
 
 # ── Audio devices (optional — leave blank for the system default) ──────────
-#MIC_DEVICE=
+# Don't guess a device name by hand — the tray's "Select microphone…" lists
+# what sounddevice actually sees and writes the pick here for you.
+MIC_DEVICE=1
 #SPEAKER_DEVICE=
 
 # ── VAD tuning (optional) ───────────────────────────────────────────────────
@@ -83,7 +85,7 @@ ELEVENLABS_VOICE_ID=
 #FOLLOW_UP_GRACE_SECONDS=7      # longer than VAD_GRACE_SECONDS: you were asked something
 
 # ── Pet window (optional) ───────────────────────────────────────────────────
-#PET_SIZE=160
+PET_SIZE=80
 #PET_START_X=
 #PET_START_Y=
 #PET_ALWAYS_ON_TOP=true
@@ -103,6 +105,9 @@ ELEVENLABS_VOICE_ID=
 #PET_CLICK_THROUGH=false
 #PET_EDGE_SNAP=true
 #PET_SNAP_MARGIN=48
+# Start where you last dragged it. A position on a monitor that's no longer
+# connected is ignored, so unplugging a screen can't hide the pet off-desktop.
+#PET_REMEMBER_POSITION=true
 
 # ── Barge-in (optional) — interrupt the pet mid-sentence ────────────────────
 # BARGE_IN_MODE decides what counts as an interruption:
@@ -138,6 +143,17 @@ ELEVENLABS_VOICE_ID=
 # ── Streaming TTS (optional) — starts talking on the first chunk ────────────
 #TTS_STREAMING=true
 
+# ── Latency: streaming the reply and the transcript ─────────────────────────
+# STREAMING_REPLIES speaks each sentence as the server generates it, instead of
+# waiting out the whole model call before the first word. STT_STREAMING sends
+# mic frames to the server's /desk/stt relay as you talk, so the transcript is
+# ready the moment you stop, instead of uploading the whole clip afterward.
+# Both fall back to the old path automatically if anything goes wrong.
+# VAD_SILENCE_END_SEC is the other half: it is dead air on every single turn,
+# so 0.8-1.0 feels markedly snappier than the 1.2 default.
+#STREAMING_REPLIES=true
+#STT_STREAMING=true
+
 # ── Screen context (optional) ───────────────────────────────────────────────
 # Sends the focused window's title along with what you said, so "what's this
 # error?" has a referent. Text only — no screenshots leave the machine.

CLAUDE.md

@@ -11,14 +11,15 @@ dependency** on the server repo; it's a standalone HTTP client configured via
 its own `.env`.
 
 Pipeline: `mic → openWakeWord ("thunderbolt", on-device) / push-to-talk /
-click → record utterance → Deepgram STT → + active-window + screen-layout
+click → record utterance → the server's own /desk/stt (same relay the
-context → POST /desk/converse → [server may relay a shell command to run on
+Android app uses — no local Deepgram account) → + active-window + screen-
-this machine, or a `petctl` pseudo-command that moves/emotes the pet, jumps it
+layout context → POST /desk/converse → [server may relay a shell command to
-to another monitor, reads a screen's text back, or plays a multi-voice scene
+run on this machine, or a `petctl` pseudo-command that moves/emotes the pet,
-instead] → reply (optionally tagged with a voice the server picked for it) →
+jumps it to another monitor, reads a screen's text back, or plays a
-ElevenLabs streaming TTS (or offline pyttsx3 fallback) → speakers`, with the
+multi-voice scene instead] → reply (optionally tagged with a voice the
-pet sprite/speech bubble reflecting state throughout, and playback
+server picked for it) → the server's own /desk/tts, streaming (or offline
-interruptible by talking over it (barge-in).
+pyttsx3 fallback) → speakers`, with the pet sprite/speech bubble reflecting
+state throughout, and playback interruptible by talking over it (barge-in).
 
 Because a relayed command's output goes back up the tool-result relay before
 the final reply, a `petctl read` mid-turn means Bolt can look at a monitor and
@@ -35,6 +36,10 @@ all suppressed while it's napping (quiet hours / fullscreen DND).
 ./run.sh          # macOS/Linux
 run.bat           # Windows
 
+# Preflight: is this install actually going to work? (config, mic, keys, sprites…)
+python -m bolt_pet --doctor          # shallow: no network, no mic
+python -m bolt_pet --doctor --deep   # contacts the server and opens the microphone
+
 # Run tests (no pytest config file — tests self-insert repo root via sys.path).
 # QT_QPA_PLATFORM=offscreen avoids a QApplication segfault on headless/no-display hosts.
 QT_QPA_PLATFORM=offscreen .venv/bin/pytest tests/
@@ -48,9 +53,12 @@ python scripts/slice_spritesheet.py path/to/sheet.png assets/sprites/idle --cols
 ```
 
 There is no lint/build step configured beyond pytest. `cp .env.example .env`
-and fill in `BOLT_SERVER_URL` / `DESK_API_KEY` (+ `DEEPGRAM_API_KEY`,
+and fill in `BOLT_SERVER_URL` / `DESK_API_KEY` before running — without them
-`ELEVENLABS_API_KEY`) before running — without server config the controller
+the controller logs a missing-config message and exits its thread instead of
-logs a missing-config message and exits its thread instead of starting.
+starting. That's also all STT and normal-reply TTS need now (both go through
+the server); `ELEVENLABS_VOICE_ID` picks the voice, and `ELEVENLABS_API_KEY`
+is only for the one feature with no server endpoint — multi-voice
+`dialoguectl` scenes, see dialogue.py below.
 
 ## Architecture
 
@@ -103,16 +111,17 @@ logs a missing-config message and exits its thread instead of starting.
   off entirely with `RECEIVE_FILES=false`.
 - **`audio/`** — `mic.py` (energy-based VAD utterance capture, ported from the
   server repo's `bolt_desk.py`), `wake_word.py` (openWakeWord `thunderbolt.onnx`
-  detection + `NearMissLog` for threshold tuning — see below), `stt.py`
+  detection + `NearMissLog` for threshold tuning — see below), `stt.py` +
-  (Deepgram), `tts.py` (ElevenLabs, streaming by default — `stream_pcm()` +
+  `stt_stream.py` (the server's own `/desk/stt` websocket relay — no local
-  `play_stream()` start playback on the first chunk; `chunks_to_int16()`
+  Deepgram account; see below), `tts.py` (the server's own `/desk/tts`,
-  carries odd bytes across HTTP chunk boundaries, without which everything
+  streaming by default — `stream_pcm()` + `play_stream()` start playback on
-  after the first split sample plays as static — falling back to whole-clip
+  the first chunk; `chunks_to_int16()` carries odd bytes across HTTP chunk
-  PCM then offline `pyttsx3`; every entry point takes an optional `voice_id`
+  boundaries, without which everything after the first split sample plays as
-  overriding `ELEVENLABS_VOICE_ID`, and `model_for()` picks the multilingual
+  static — falling back to whole-clip PCM then offline `pyttsx3`; every entry
-  model whenever there's an override or non-ASCII text, since the default
+  point takes an optional `voice_id` overriding `ELEVENLABS_VOICE_ID` — which
-  `eleven_flash_v2` is English-only and would read either as garbled
+  model to synthesize with is the server's call now, not this client's;
-  phonetic English rather than failing), `barge_in.py` (two detectors behind one
+  see `synthesize_dialogue()` further down for the one path that's still
+  ElevenLabs-direct), `barge_in.py` (two detectors behind one
   `reset()`/`check()` shape, chosen by `BARGE_IN_MODE` via `make_detector`:
   **wake** (default) scores every frame with the same openWakeWord model the
   idle listener uses, so only the wake phrase cuts playback; **energy** is the
@@ -239,7 +248,18 @@ logs a missing-config message and exits its thread instead of starting.
 - **`quiet.py`** — quiet-hours spec parsing (`23:00-08:00`, wraps midnight,
   comma-separated). Napping suppresses *proactive* noise and wandering only;
   wake word / click / push-to-talk still work.
-- **`notifications.py`** — Linux/D-Bus notification bridge: tails
+- **`notifications.py`** — Linux/D-Bus notification bridge. Two latency/loss bugs
+  fixed 2026-08-02, both in `controller._maybe_heartbeat`: draining was wired to the
+  60s heartbeat interval rather than the ~1.2s wake tick, and a heartbeat that landed
+  mid-conversation stamped its own clock *before* checking — burning the slot and
+  waiting another full interval, repeatedly, which is how a notification could go
+  unspoken for five or ten minutes. Draining now runs on every tick while IDLE, and the
+  heartbeat clock only advances when the heartbeat actually runs. Separately, the rate
+  limit used to *drop* notifications inside its window (a second text a minute later was
+  silently lost); the filter still gates at queue time but the limit is gone — a burst
+  is **batched into one turn** instead, same single round trip, no lost messages, capped
+  by `_MAX_PENDING_NOTIFICATIONS`.
+- **`notifications.py` internals** — tails
   `dbus-monitor`, parses Notify calls (pure `iter_notifications()`), filters
   and rate-limits them (`NotificationGate`), and the controller forwards
   survivors through `converse()`. Off by default — each one is a round trip.
@@ -275,6 +295,93 @@ logs a missing-config message and exits its thread instead of starting.
 - **`hotkey.py`** — global push-to-talk via `pynput`; soft-fails with a logged
   reason (Wayland, missing package, macOS permissions) since the wake word is
   the primary trigger.
+- **`audio/stt_stream.py`** / **`audio/stt.py`** — speech-to-text via the
+  server's `/desk/stt` websocket relay (audio up, Deepgram's JSON messages
+  down untouched — the same relay the Android app uses; no local Deepgram
+  account or API key). `stt_stream.py` is the opportunistic optimisation:
+  the one-shot path waits for the utterance to end, then sends the whole
+  clip and waits again; that second wait is dead time that grows with how
+  long you spoke, and streaming removes it — `record_utterance(on_frame=...)`
+  hands each captured frame to a `StreamingTranscriber` as it's captured, so
+  by the time the VAD decides you stopped the transcript is essentially
+  already there. `stt.py`'s `transcribe()` is the *guaranteed* fallback for
+  when that didn't produce anything: since there's no separate REST endpoint
+  server-side, it opens the exact same relay via `stt_stream.connect()` and
+  just feeds the whole buffered utterance in one go — deliberately
+  *unconditional*, not gated by `STT_STREAMING`/`available()` the way the
+  opportunistic path is, since there's nothing left to fall back to if that
+  connection fails. Three more deliberate limits on the streaming half:
+  `open()` returning **None is an ordinary outcome** (no websocket-client, no
+  network, streaming turned off) because the full audio is still buffered
+  and `controller._transcribe` just falls back to `stt.transcribe()`; the
+  **local VAD still decides when you stopped** rather than Deepgram's
+  endpointing, since barge-in, follow-up listening and the grace period are
+  all built on it and coupling them to the network is not a first-pass
+  change; and the socket is **per-utterance**, because holding one open
+  across an idle pet bills for silence and dies on the first blip.
+  Off: `STT_STREAMING=false`.
+- **Streamed replies** — `server_client.converse_stream()` reads NDJSON from the desk
+  API's `/desk/converse_stream` and speaks each sentence as it arrives
+  (`controller._speak_stream_chunk`), so the wait is time-to-first-sentence instead of
+  the whole model call. `Reply.spoken` marks a reply whose sentences were already said:
+  the text is still carried, because the follow-up rule needs to see whether it ended on
+  a question, but it must not be read out again. A stream that fails *before* anything
+  was spoken falls back to `converse()` invisibly; one that fails after ends the turn
+  quietly rather than repeating the first half. Off: `STREAMING_REPLIES=false`.
+- **`speech.py`** — everything the pet says, and the policy differences between
+  kinds of saying. There were four near-copies of this in `controller.py` (a
+  reply, a holding line, a streamed sentence, a dialogue scene), each repeating
+  the same dance — transition state, show the bubble, maybe record history,
+  reset barge-in, call TTS, reset barge-in *again*, resume state, decide
+  whether to keep the mic open — and they had already drifted apart: one forgot
+  to arm barge-in, another skipped the follow-up rule. Now the dance is
+  `Speaker.say()` and the differences are data on a frozen `Utterance`
+  (`record`, `resume`, `hold_talking`, `follow_up`, `interruptible`), built by
+  the four classmethods `reply`/`holding`/`stream_chunk`/`scene`. Notable
+  policies: a holding line is **not** recorded (it's filler; the transcript
+  should keep the answer) and **not** interruptible (cutting off "give me a
+  sec" strands the tool already running), and it resumes the state it
+  interrupted rather than dropping to IDLE, because the turn isn't over. A
+  streamed chunk stays TALKING so the sprite doesn't flicker between sentences.
+  Collaborators are injected, so all of this is tested without Qt, audio or a
+  real state machine. Two subtleties that are bugs waiting to happen: the
+  barge-in detector is read through a **callable**, not held (it's built after
+  the Speaker — it needs the mic stream — and swapped when the mode changes; two
+  copies drifting apart is invisible until the wake model starts hearing the
+  pet), and `last_detail` is captured *before* the post-playback reset, since
+  reading it after means every interruption reports zeroed counters.
+  `follow_up_decision()` is the mic-open rule as a pure function.
+- **Lip-sync** — the mouth is driven by the audio, not a timer.
+  `audio/tts.level_of(frame)` reduces a PCM frame to 0..1 loudness on a **sqrt
+  curve** (speech sits well below peak most of the time, so a linear map leaves
+  the mouth barely open during normal talking), `envelope()` does the same for a
+  whole clip, and playback calls the `on_level` hook per chunk. That travels
+  `Speaker` → `controller.mouth` (a Signal) → `PetWindow.set_mouth`, and
+  `_mouth_frame()` indexes the talking frames directly — which works because
+  `generate_bolt_sprites.py` draws them as an **openness ramp** (closed first,
+  widest last) rather than an arbitrary loop. Levels going stale
+  (`_MOUTH_STALE_SECONDS`) hands control back to the ordinary animation, so
+  offline TTS — which has no envelope — degrades to the timed loop instead of
+  freezing the mouth mid-syllable.
+- **`window_state.py`** — where the pet was left, so it starts there.
+  `~/.cache/bolt-pet/window.json`, atomic write on drag-end, every function
+  swallows its own errors (a corrupt state file must mean the default corner,
+  never a pet that won't start). `is_visible_on()` re-validates against the
+  *current* screen layout on load, because the common case for a stale position
+  is exactly the dangerous one: the pet was last on a monitor that is now
+  unplugged, and restoring it faithfully puts it somewhere unreachable. Takes
+  plain rectangles rather than importing Qt. Off: `PET_REMEMBER_POSITION=false`.
+- **`doctor.py`** — `python -m bolt_pet --doctor`, a preflight, written after a
+  week of debugging things one command would have shown: a venv whose python
+  was a zero-byte file, an OCR engine never installed, a wrong proxy header.
+  Twelve independent checks, each reporting ok / warn (degraded but working) /
+  fail, and each saying **what to do about it** — `screen reading: warn` is
+  useless alone, `apt install tesseract-ocr` is the whole point. Nothing raises:
+  a doctor that crashes on a broken install is diagnosing the wrong patient, so
+  `run()` catches per-check and a failed check becomes a FAIL row rather than a
+  traceback. Shallow by default (no network, no mic) since it's the first thing
+  you reach for when the network is what's broken; `--deep` actually contacts
+  the server and opens the microphone.
 - **`speech_text.py`** — sanitizes server replies before they're heard/shown.
   `for_speech()` (called inside `tts.speak()`, so every path to the speakers is
   covered) strips markdown, emoji, URLs and stray symbols the voice would read
@@ -284,7 +391,7 @@ logs a missing-config message and exits its thread instead of starting.
   whether a reply leaves the pet waiting on an answer: it tests the *spoken*
   form (so a '?' inside a stripped code block or URL doesn't count) and only a
   trailing one counts, since a question asked in passing isn't awaiting a
-  reply. `controller._should_follow_up` uses it to keep listening without the
+  reply. `speech.follow_up_decision` uses it to keep listening without the
   wake word, capped by `FOLLOW_UP_MAX_TURNS` so a server that ends every reply
   with a question can't loop forever off mic noise. Pure string logic, no
   Qt/audio imports.
@@ -414,6 +521,20 @@ HTTP chunk reassembly by `chunks_to_int16`, emote motion by
 `screen_context.context_for` / `is_fullscreen_active`, otherwise they shell
 out to xprop on a headless box.
 
+**`test_pipeline_smoke.py` is the exception, deliberately.** Unit tests inject
+a fake at the seam they care about, and a run of production bugs — notifications
+sitting unspoken for minutes, the pet saying things twice, `[laughing]` read
+aloud, a device command arriving as prose — got through with every one of them
+green, because each was an *interaction* between two individually-correct
+units. So that file stands up a real threaded HTTP server on a loopback port,
+speaks the desk protocol at it, and drives whole turns through the real
+`server_client` (including NDJSON streaming), the real controller and the real
+state machine, faking only the mic stream and the speakers. When a bug crosses
+a module boundary, add the case there; when it lives inside one module, the
+pure-function pattern above is still the cheaper test. It is also worth
+mutation-checking a new case — break the source line it is meant to catch and
+confirm it actually goes red.
+
 ## Security notes
 
 The server can relay a shell command back to this machine to execute as the

README.md

@@ -3,16 +3,19 @@
 A little animated pet that lives on your desktop and is just a face on top
 of your Bolt server — same brain, memory, tools, and persona as Discord
 chat and the Linux desk client. It talks to `ai/desk_api.py` on the server
-exactly the way `desk_client/bolt_desk.py` does; this project only adds the
+exactly the way `desk_client/bolt_desk.py` does — including speech: STT and
-on-screen pet and swaps Deepgram/ElevenLabs playback to be cross-platform
+TTS are the server's own `/desk/stt` and `/desk/tts`, the same endpoints the
-(no `mpv`/`ffplay`/`espeak-ng` subprocess calls — pure `sounddevice`).
+Android app uses, so there's no separate Deepgram or ElevenLabs account to
+set up for the pet to talk. Playback is cross-platform (no `mpv`/`ffplay`/
+`espeak-ng` subprocess calls — pure `sounddevice`).
 
 ```
 mic → wake-phrase spotter ("thunderbolt") / hotkey / click → record utterance
-    → Deepgram STT (+ the focused window's title, for "what's this error?")
+    → server-hosted STT (+ the focused window's title, for "what's this
-    → POST /desk/converse on your Bolt server → [server may relay a shell
+      error?") → POST /desk/converse on your Bolt server → [server may relay
-      command back to run on THIS machine, or a `petctl` command that moves
+      a shell command back to run on THIS machine, or a `petctl` command
-      or emotes the pet] → reply → ElevenLabs streaming TTS → speakers
+      that moves or emotes the pet] → reply → server-hosted streaming TTS
+      → speakers
     → shown in a speech bubble + the pet's sprite state (idle/listening/
       thinking/talking) updates the whole time
 ```
@@ -36,10 +39,13 @@ all, so it can be copied anywhere and configured with its own `.env`.
    - `BOLT_SERVER_URL` + `DESK_API_KEY` — same as `desk_client/.env` on the
      server side. Use the server's master `DESK_API_KEY`, or mint yourself a
      personal one via the desk-only `api_key_generate` marker (see the main
-     repo's `CLAUDE.md` → "Per-user API keys").
+     repo's `CLAUDE.md` → "Per-user API keys"). That's it for STT — no
-   - `DEEPGRAM_API_KEY` for STT.
+     Deepgram account needed, it goes through the server's own `/desk/stt`.
-   - `ELEVENLABS_API_KEY` + `ELEVENLABS_VOICE_ID` for TTS (optional — falls
+   - `ELEVENLABS_VOICE_ID` for TTS (optional — falls back to offline TTS via
-     back to offline TTS via `pyttsx3` if omitted or if a request fails).
+     `pyttsx3` if omitted or if the server call fails). No local ElevenLabs
+     API key needed for this either; `ELEVENLABS_API_KEY` is only for the
+     multi-voice `dialoguectl` scenes further down, the one feature the
+     server has no endpoint for.
 3. Run it:
    - macOS/Linux: `./run.sh`
    - Windows: `run.bat`
@@ -96,9 +102,10 @@ Ask for a different voice — "use a clearer voice", "talk like a pirate", "say
 that in Japanese" — and Bolt searches the ElevenLabs voice library on the
 server, picks one, and tags his reply with it (`speak_as`); the pet is what
 actually speaks in it. A Voice Library pick is added to your ElevenLabs
-account automatically the first time it's used, and non-English replies (or
+account automatically the first time it's used. The server also synthesizes
-any picked voice) go through `ELEVENLABS_MULTILINGUAL_MODEL_ID` rather than
+with one fixed, multilingual-capable model for every request now (not a
-the English-only `eleven_flash_v2` default.
+flash/multilingual switch per reply) — nothing to configure on this side,
+and non-English text or a picked voice no longer needs special-casing here.
 
 The new voice **stays on** for the rest of the conversation, because the
 server tags a single reply and doesn't remember which voice it chose — so
@@ -140,6 +147,33 @@ near misses — frames that scored just under the threshold — and the slider
 takes effect immediately, mid-listen. Set the threshold just below the peak
 you can hit reliably, then write it into `.env` as `WAKE_WORD_THRESHOLD`.
 
+## Something not working?
+
+```bash
+python -m bolt_pet --doctor
+```
+
+Checks the things that make the pet look broken in ways that don't point at
+themselves — missing server config (the controller exits its thread at startup,
+so the pet appears alive and simply never answers), no input device, no OCR
+engine behind `petctl read`, a wake model that isn't where `.env` says, a
+silence timeout long enough to feel like lag. Each line says what to do about
+it, not just what's wrong. It doesn't touch the network or open the microphone
+unless you add `--deep`, so it's safe to run when the network is the suspect.
+
+## Little things
+
+The pet **remembers where you left it** — drag it somewhere deliberate and
+that's where it starts next time. If that position is on a monitor you've since
+unplugged it goes back to the default corner rather than restoring itself
+somewhere off-screen. `PET_REMEMBER_POSITION=false` to always start in the
+corner.
+
+Its **mouth moves with the actual audio** rather than flapping on a timer: the
+PCM going to the speakers is reduced to a loudness per frame and that picks the
+talking sprite, so the pet shuts up when the voice pauses. Offline `pyttsx3`
+playback has no waveform to follow, so it falls back to the timed loop.
+
 ## Project layout
 
 ```
@@ -148,18 +182,23 @@ bolt_pet/
   state.py            PetState enum + a small transition-checked state machine
   server_client.py    /desk/converse, /desk/tool_result, /desk/report_status
   controller.py        the pipeline: wake word -> STT -> server -> TTS, on a QThread
-  speech_text.py        strips markdown/emoji/URLs so the voice never says "asterisk"
+  speech.py             what the pet says and how each kind of saying behaves
+  speech_text.py         strips markdown/emoji/URLs so the voice never says "asterisk"
   pet_actions.py         petctl move/emote/say/wander/nap parsing
   screen_context.py       active-window title + fullscreen detection
   quiet.py                 quiet-hours schedule
   notifications.py          desktop notification bridge (Linux/D-Bus)
   history.py                 rolling conversation transcript
   hotkey.py                   global push-to-talk (pynput, optional)
+  window_state.py              remembers where you left the pet
+  doctor.py                     `--doctor` preflight: is this install going to work?
   audio/
     mic.py             input stream + energy-based VAD utterance capture
     wake_word.py        openWakeWord thunderbolt.onnx detection (see above)
-    stt.py               Deepgram
+    stt.py               server /desk/stt relay (one-shot: whole utterance at once)
-    tts.py                ElevenLabs streaming PCM, offline pyttsx3 fallback
+    stt_stream.py         server /desk/stt live websocket — transcribes while you speak
+    tts.py                 server /desk/tts streaming PCM, offline pyttsx3 fallback,
+                            plus the loudness envelope that drives the mouth
     barge_in.py            "you started talking" detector, to cut playback short
   ui/
     app.py              wires QApplication + window + tray + controller thread together
@@ -172,8 +211,10 @@ bolt_pet/
 scripts/
   slice_spritesheet.py  cuts a grid sprite sheet into the per-frame convention
 tests/                  pure-logic unit tests (state machine, wake-phrase
-                         matching, HTTP client against mocks) — nothing here
+                         matching, HTTP client against mocks) plus one
-                         needs real audio hardware or a display
+                         end-to-end smoke test that drives whole turns against
+                         a real local HTTP server — nothing here needs real
+                         audio hardware or a display
 ```
 
 ## Security notes
@@ -197,7 +238,6 @@ notifications. No screenshots or images are ever sent.
 
 ## Known limitations / not-yet-done
 
-- Pet screen position isn't persisted across restarts.
 - Wandering is a straight walk to a random point — no Shimeji-style physics,
   wall-climbing or falling.
 - Push-to-talk and the notification bridge are platform-limited: the hotkey

bolt_pet/__main__.py

@@ -1,8 +1,15 @@
-"""Entry point: python -m bolt_pet"""
+"""Entry point: python -m bolt_pet [--doctor [--deep]]"""
 
 import sys
 
-from .ui.app import run
-
 if __name__ == "__main__":
+    if "--doctor" in sys.argv:
+        # Imported inside the branch, not at module scope: the doctor exists to
+        # diagnose installs where importing the UI would itself blow up.
+        from .doctor import main as doctor
+
+        sys.exit(doctor(sys.argv[1:]))
+
+    from .ui.app import run
+
     sys.exit(run())

bolt_pet/audio/mic.py

@@ -37,6 +37,20 @@ def rms(frame: np.ndarray) -> float:
     return float(np.sqrt(np.mean(frame.astype(np.float64) ** 2)))
 
 
+def _emit(on_frame, frame) -> None:
+    """Hand a frame to a listener without letting it break the capture.
+
+    Streaming STT is an optimisation riding along with recording; if its
+    socket dies mid-utterance the recording must carry on untouched, because
+    the one-shot fallback is about to need the full buffer."""
+    if on_frame is None:
+        return
+    try:
+        on_frame(frame)
+    except Exception:
+        pass
+
+
 def record_utterance(
     stream: AudioStream,
     should_continue=lambda: True,
@@ -47,6 +61,7 @@ def record_utterance(
     grace_s: float = None,
     frame_len: int = config.FRAME_LEN,
     sample_rate: int = config.SAMPLE_RATE,
+    on_frame=None,
 ) -> Optional[np.ndarray]:
     """Capture one utterance from *stream*: wait for speech to start, stop
     after trailing silence. Returns None if nothing usable was heard.
@@ -55,6 +70,11 @@ def record_utterance(
     (e.g. the pet window was closed) without needing threading primitives
     baked into this function.
 
+    *on_frame*, if given, is called with each captured frame while speech is
+    in progress — that is how streaming STT transcribes as you talk rather
+    than after (see audio/stt_stream.py). It is fire-and-forget: this function
+    still returns the full buffer, so a failed stream costs nothing.
+
     *grace_s* is how long to wait for speech to *begin* before giving up.
     The controller stretches it for follow-up questions, where you're being
     asked something and need a moment to think rather than having just said
@@ -83,10 +103,12 @@ def record_utterance(
             if frame_rms >= rms_threshold:
                 started = True
                 frames.append(frame)
+                _emit(on_frame, frame)
             elif waited > grace_frames:
                 return None  # woke it up but said nothing
             continue
         frames.append(frame)
+        _emit(on_frame, frame)
         if frame_rms < rms_threshold:
             silence_frames += 1
             if silence_frames >= silence_limit:
@@ -104,6 +126,27 @@ def record_utterance(
     return pcm
 
 
+def list_input_devices() -> list[dict]:
+    """Every input-capable device sounddevice can see, as
+    {"index", "name", "hostapi"}.
+
+    On Windows in particular, the *same* physical (or virtual) device is
+    normally listed once per host API — MME, DirectSound, WASAPI, WDM-KS —
+    all sharing (or near enough sharing) one name. That makes the name alone
+    an unreliable, often ambiguous way to pick a device (sounddevice matches
+    a string device as a substring and refuses to guess between ties), so the
+    tray's mic picker saves the numeric index instead, which this list exists
+    to let it show alongside a human-readable label."""
+    import sounddevice as sd
+
+    devices = []
+    for index, info in enumerate(sd.query_devices()):
+        if info.get("max_input_channels", 0) > 0:
+            hostapi = sd.query_hostapis(info["hostapi"])["name"]
+            devices.append({"index": index, "name": info["name"], "hostapi": hostapi})
+    return devices
+
+
 def open_input_stream():
     """Real sounddevice input stream, imported lazily so pure-logic tests
     (record_utterance with a fake stream) don't need PortAudio installed."""

bolt_pet/audio/stt.py

@@ -1,14 +1,20 @@
 """Speech-to-text for the actual query, after the wake word fires.
 
-Deepgram, same as desk_client/bolt_desk.py.
+The "guaranteed" fallback for when stt_stream's opportunistic live-feed
+session didn't produce a transcript (streaming disabled, or the socket
+never came up). There is no separate one-shot REST endpoint server-side any
+more — /desk/stt is a websocket relay only — so this connects the exact
+same way stt_stream.py does and just feeds the whole buffered utterance in
+one go instead of frame-by-frame as it's captured. That connection is
+unconditional (not gated by STT_STREAMING, which only controls the
+opportunistic optimisation), since there is nothing left to fall back to
+if it fails.
 """
 
 from __future__ import annotations
 
-import requests
-
 from .. import config
-from .mic import pcm_to_wav_bytes
+from . import stt_stream
 
 
 class SttError(Exception):
@@ -16,27 +22,12 @@ class SttError(Exception):
 
 
 def transcribe(pcm) -> str:
-    if not config.DEEPGRAM_API_KEY:
+    if not config.is_configured():
-        raise SttError("DEEPGRAM_API_KEY is not set")
+        raise SttError("BOLT_SERVER_URL / DESK_API_KEY not set")
     try:
-        response = requests.post(
+        socket = stt_stream.connect()
-            "https://api.deepgram.com/v1/listen",
-            params={"model": config.DEEPGRAM_MODEL, "language": "en", "smart_format": "true"},
-            headers={
-                "Authorization": f"Token {config.DEEPGRAM_API_KEY}",
-                "Content-Type": "audio/wav",
-            },
-            data=pcm_to_wav_bytes(pcm),
-            timeout=30,
-        )
-        response.raise_for_status()
     except Exception as exc:
-        raise SttError(f"transcription request failed: {exc}") from exc
+        raise SttError(f"couldn't reach the transcription server: {exc}") from exc
-    try:
+    session = stt_stream.StreamingTranscriber(socket)
-        return (
+    session.feed(pcm)
-            response.json()
+    return session.finish()
-            .get("results", {}).get("channels", [{}])[0]
-            .get("alternatives", [{}])[0].get("transcript", "")
-        ).strip()
-    except Exception as exc:
-        raise SttError(f"couldn't parse transcription response: {exc}") from exc

bolt_pet/audio/stt_stream.py

@@ -0,0 +1,202 @@
+"""Streaming speech-to-text — transcribing *while* you talk, not after.
+
+The one-shot path (`stt.transcribe`) waits for the utterance to finish, then
+sends the whole clip and waits again. That second wait is dead time between
+you stopping and the pet reacting, and it grows with the length of what you
+said — a thirty-second question costs noticeably more than a five-second one.
+
+This connects to the Bolt server's own `/desk/stt` — the same websocket relay
+the Android app uses — which forwards audio to Deepgram and Deepgram's JSON
+messages back untouched. Frames go up as they are captured, so by the time the
+VAD decides you have stopped, the transcript is essentially already there.
+There is no local Deepgram account or API key any more; auth is this pet's own
+`DESK_API_KEY`, same as every other call to the server.
+
+Design constraints that shaped this:
+
+- **Failure must be invisible.** No websocket, no network, a mid-utterance
+  disconnect — all of it falls back to the one-shot path, which still has the
+  full audio buffered. Streaming is an optimisation here, never a dependency —
+  `available()`/`open()` returning None/False is an ordinary outcome, not an
+  error. `stt.transcribe()` (the *guaranteed* fallback) talks to the exact same
+  server relay via `connect()` directly, bypassing that opportunistic gate,
+  since there is no second, different backend left to fall back to.
+- **The VAD still decides when you stopped.** Deepgram has its own endpointing
+  and using it would save more, but it would also move a decision the rest of
+  the pipeline is built around (barge-in, follow-up listening, the grace
+  period) into a remote service. Not worth coupling those to the network on
+  the first pass.
+- **The socket is per-utterance.** Holding one open across an idle pet would
+  bill for silence and drop on the first network blip; opening one takes
+  ~100ms, which is already inside the time it takes a person to start talking.
+
+The websocket client is injectable, so the whole protocol — send frames, read
+`is_final` transcripts, close, take the result — is tested without a network.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+from typing import Callable, Optional
+
+import numpy as np
+
+from .. import config
+
+logger = logging.getLogger("bolt_pet.stt_stream")
+
+
+def available() -> bool:
+    """Whether the streaming (transcribe-while-talking) optimization should
+    be attempted opportunistically. Not a gate on transcription itself — the
+    server relay is the only way to transcribe at all now, so
+    stt.transcribe() connects via connect() directly rather than through
+    this, and isn't affected by STT_STREAMING being off."""
+    if not config.STT_STREAMING or not config.is_configured():
+        return False
+    try:
+        import websocket  # noqa: F401  (websocket-client)
+        return True
+    except Exception:
+        return False
+
+
+def _connect(sample_rate: int = None):
+    """Open a websocket to the server's `/desk/stt` relay. Raises on any
+    failure — this is the "no fallback left" connector `stt.transcribe()`
+    uses directly, as well as the default for `StreamingTranscriber.open()`.
+
+    The server chooses the STT model and the endpointing behaviour; this
+    only states the audio format about to be sent, which is fixed by the
+    wake model upstream of it."""
+    import websocket
+
+    rate = sample_rate or config.SAMPLE_RATE
+    url = (
+        config.SERVER_URL.replace("http", "ws", 1)
+        + "/desk/stt"
+        + f"?session_id={config.SESSION_ID}&encoding=linear16&sample_rate={rate}"
+    )
+    return websocket.create_connection(
+        url, header=[f"X-Desk-Api-Key: {config.API_KEY}"], timeout=10,
+    )
+
+
+# Public name for external callers (stt.py, tests) — named separately from
+# the module-private def so StreamingTranscriber.open()'s `connect` parameter
+# can shadow the bare name locally without losing access to this.
+connect = _connect
+
+
+class StreamingTranscriber:
+    """One utterance's worth of live transcription.
+
+    Usage mirrors how the capture loop already works — feed frames as they
+    arrive, then ask what was said:
+
+        session = StreamingTranscriber.open()
+        ...
+        session.feed(frame)      # per mic frame, non-blocking
+        text = session.finish()  # after the VAD says you stopped
+    """
+
+    def __init__(self, socket, *, sample_rate: int = None):
+        self._socket = socket
+        self._sample_rate = sample_rate or config.SAMPLE_RATE
+        self._transcript: list[str] = []
+        self._lock = threading.Lock()
+        self._closed = False
+        self._reader = threading.Thread(
+            target=self._read_loop, name="stt-stream-reader", daemon=True)
+        self._reader.start()
+
+    # -- lifecycle ----------------------------------------------------------
+
+    @classmethod
+    def open(cls, *, connect: Optional[Callable] = None,
+             sample_rate: int = None) -> Optional["StreamingTranscriber"]:
+        """Connect, or return None if streaming isn't possible right now.
+
+        None is a normal outcome, not a failure: the caller keeps the audio and
+        falls back to the one-shot upload."""
+        if connect is None and not available():
+            return None
+        rate = sample_rate or config.SAMPLE_RATE
+        try:
+            socket = connect() if connect is not None else _connect(rate)
+            return cls(socket, sample_rate=rate)
+        except Exception as exc:
+            logger.info("Streaming STT unavailable (%s) — using the one-shot path.", exc)
+            return None
+
+    def feed(self, frame: np.ndarray) -> None:
+        """Send one captured frame. Never raises — a dead socket just means the
+        fallback will do the work."""
+        if self._closed:
+            return
+        try:
+            self._socket.send_binary(np.asarray(frame, dtype=np.int16).tobytes())
+        except Exception:
+            logger.debug("Streaming STT send failed; abandoning the stream", exc_info=True)
+            self._closed = True
+
+    def finish(self, timeout: float = 3.0) -> str:
+        """Close the stream and return whatever was transcribed.
+
+        Deepgram flushes its final results after the close frame, so this waits
+        briefly for the reader — bounded, because a hung socket must not hold
+        up the reply."""
+        if not self._closed:
+            try:
+                self._socket.send(json.dumps({"type": "CloseStream"}))
+            except Exception:
+                pass
+        self._closed = True
+        self._reader.join(timeout=timeout)
+        try:
+            self._socket.close()
+        except Exception:
+            pass
+        with self._lock:
+            return " ".join(part for part in self._transcript if part).strip()
+
+    # -- the reader ---------------------------------------------------------
+
+    def _read_loop(self) -> None:
+        while not self._closed:
+            try:
+                message = self._socket.recv()
+            except Exception:
+                break
+            if not message:
+                break
+            text, is_final = self._parse(message)
+            if text and is_final:
+                with self._lock:
+                    self._transcript.append(text)
+
+    @staticmethod
+    def _parse(message) -> tuple[str, bool]:
+        """Pull (text, is_final) out of a Deepgram results frame.
+
+        Tolerant on purpose: anything unrecognised is ignored rather than
+        raising on the reader thread, where an exception would silently kill
+        transcription for the rest of the utterance."""
+        try:
+            if isinstance(message, bytes):
+                message = message.decode("utf-8", "ignore")
+            data = json.loads(message)
+        except (TypeError, ValueError):
+            return "", False
+        if not isinstance(data, dict):
+            return "", False
+        alternatives = (
+            ((data.get("channel") or {}).get("alternatives") or [])
+            if data.get("type") in (None, "Results") else []
+        )
+        if not alternatives:
+            return "", False
+        text = str((alternatives[0] or {}).get("transcript") or "").strip()
+        return text, bool(data.get("is_final") or data.get("speech_final"))

bolt_pet/audio/tts.py

@@ -1,19 +1,29 @@
-"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just
+"""Text-to-speech: the Bolt server's own `/desk/tts` — the same endpoint the
+Android app streams from — requested as raw PCM so playback is just
 sounddevice — no external player binary (mpv/ffplay), unlike
 desk_client/bolt_desk.py which shells out because it only targets Linux.
 
-Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
+No local ElevenLabs account needed for this: the server picks the voice
-on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
+(ELEVENLABS_VOICE_ID, or a `speak_as` override) and the synthesis model
-fails, so the pet can still talk with zero cloud config.
+itself, authenticated with this pet's own DESK_API_KEY. Falls back to
+pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech on macOS,
+espeak on Linux) if the server call fails, so the pet can still talk even
+with the server unreachable.
 
 Every entry point takes an optional *voice_id* that overrides
 `ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as`
 reply marker reaches the speakers (see controller._apply_voice). The offline
 fallback has no such concept and always sounds like itself.
+
+`synthesize_dialogue()` below is the one exception: multi-voice
+`dialoguectl` scenes have no server endpoint, so that one call still goes
+to ElevenLabs' Text to Dialogue API directly and still needs
+ELEVENLABS_API_KEY — see dialogue.py.
 """
 
 from __future__ import annotations
 
+import time
 from typing import Iterable, Iterator, Optional
 
 import numpy as np
@@ -32,19 +42,8 @@ def voice_for(voice_id: Optional[str] = None) -> str:
     return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID
 
 
-def model_for(text: str, voice_id: Optional[str] = None) -> str:
+def _headers() -> dict:
-    """Which ElevenLabs model to synthesize with.
+    return {"X-Desk-Api-Key": config.API_KEY}
-
-    The default (`eleven_flash_v2`) is English-only, and both things that
-    reach this branch mean the reply probably isn't English: a voice the
-    server picked mid-conversation is nearly always about a language or an
-    accent, and non-ASCII text can't be English at all. Rendering either one
-    through the English model gets you a mangled phonetic reading rather
-    than a failure, which is worse — so those go through the multilingual
-    model instead."""
-    if (voice_id or "").strip() or not text.isascii():
-        return config.ELEVENLABS_MULTILINGUAL_MODEL_ID
-    return config.ELEVENLABS_MODEL_ID
 
 
 def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]:
@@ -52,48 +51,46 @@ def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarra
     callers should fall back to speak_offline() rather than treating this
     as fatal."""
     voice = voice_for(voice_id)
-    if not (config.ELEVENLABS_API_KEY and voice):
+    if not (config.is_configured() and voice):
-        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+        raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
     try:
         response = requests.post(
-            f"https://api.elevenlabs.io/v1/text-to-speech/{voice}",
+            f"{config.SERVER_URL}/desk/tts",
-            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            headers=_headers(),
-            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
-            json={"text": text, "model_id": model_for(text, voice_id)},
             timeout=60,
         )
         response.raise_for_status()
     except Exception as exc:
-        raise TtsError(f"ElevenLabs request failed: {exc}") from exc
+        raise TtsError(f"server tts request failed: {exc}") from exc
     pcm = np.frombuffer(response.content, dtype=np.int16)
     if pcm.size == 0:
-        raise TtsError("ElevenLabs returned no audio")
+        raise TtsError("server returned no audio")
     return pcm, config.TTS_SAMPLE_RATE
 
 
 def stream_pcm(
     text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None
 ) -> Iterator[np.ndarray]:
-    """Same audio as synthesize_pcm(), but yielded as it arrives from
+    """Same audio as synthesize_pcm(), but yielded as it arrives from the
-    ElevenLabs' /stream endpoint so playback can start on the first chunk
+    server so playback can start on the first chunk instead of after the
-    (~300ms) instead of after the whole clip is synthesized. Raises TtsError
+    whole clip is synthesized. Raises TtsError before yielding anything if
-    before yielding anything if the request itself fails, so callers can fall
+    the request itself fails, so callers can fall back cleanly; a mid-stream
-    back cleanly; a mid-stream failure just ends the generator."""
+    failure just ends the generator."""
     voice = voice_for(voice_id)
-    if not (config.ELEVENLABS_API_KEY and voice):
+    if not (config.is_configured() and voice):
-        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+        raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
     try:
         response = requests.post(
-            f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream",
+            f"{config.SERVER_URL}/desk/tts",
-            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            headers=_headers(),
-            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
-            json={"text": text, "model_id": model_for(text, voice_id)},
             timeout=60,
             stream=True,
         )
         response.raise_for_status()
     except Exception as exc:
-        raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc
+        raise TtsError(f"server tts stream request failed: {exc}") from exc
     return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes))
 
 
@@ -162,32 +159,81 @@ def synthesize_dialogue(
     return pcm, config.TTS_SAMPLE_RATE
 
 
-def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool:
+# ── how loud is it right now ────────────────────────────────────────────────
+# The PCM is already decoded here on its way to the speakers, so the amplitude
+# envelope is free — and it is exactly what a mouth needs to move in time with
+# speech. Throwing it away and animating the mouth on a timer instead is why
+# most talking sprites look dubbed.
+
+# int16 RMS that counts as "mouth fully open". Speech peaks around 8-12k;
+# 6000 keeps normal talking in the upper half of the range without clipping
+# every syllable to wide-open.
+_LOUD_RMS = 6000.0
+
+
+def level_of(frame: np.ndarray) -> float:
+    """0..1 loudness for one chunk of PCM.
+
+    Square-rooted because perceived loudness is not linear in amplitude — a
+    linear mapping leaves the mouth barely moving through ordinary speech."""
+    if frame is None or len(frame) == 0:
+        return 0.0
+    rms = float(np.sqrt(np.mean(np.square(frame.astype(np.float32)))))
+    return float(min(1.0, (rms / _LOUD_RMS) ** 0.5))
+
+
+def envelope(pcm: np.ndarray, sample_rate: int, fps: int = 30) -> list:
+    """Per-frame loudness for a whole clip, for playback that isn't streamed."""
+    if pcm is None or len(pcm) == 0:
+        return []
+    window = max(1, int(sample_rate / max(1, fps)))
+    return [level_of(pcm[start:start + window]) for start in range(0, len(pcm), window)]
+
+
+def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None,
+             on_level=None) -> bool:
     """Play a whole clip. Returns True if it finished, False if *should_stop*
     (barge-in) cut it short. *should_stop* is polled while audio plays — each
     poll consumes one mic frame, which is what paces this loop."""
     import sounddevice as sd
 
+    levels = envelope(pcm, sample_rate) if on_level is not None else []
+    started = time.monotonic()
     sd.play(pcm, samplerate=sample_rate, device=config.SPEAKER_DEVICE)
     if not blocking:
         return True
-    if should_stop is None:
+    if should_stop is None and on_level is None:
         sd.wait()
         return True
     while True:
+        if levels:
+            # Indexed by elapsed time rather than by chunk, because this path
+            # hands the whole clip to the device at once and never sees it
+            # again — wall clock is the only position we have.
+            index = int((time.monotonic() - started) * 30)
+            if index < len(levels):
+                try:
+                    on_level(levels[index])
+                except Exception:
+                    levels = []
         try:
             if not sd.get_stream().active:
                 break
         except Exception:
             break  # stream already torn down — playback is over
-        if should_stop():
+        if should_stop is not None and should_stop():
             sd.stop()
             return False
     return True
 
 
-def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None) -> bool:
+def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None,
-    """Play int16 chunks as they arrive. Returns False if interrupted."""
+                on_level=None) -> bool:
+    """Play int16 chunks as they arrive. Returns False if interrupted.
+
+    *on_level* receives each chunk's loudness (0..1) just before it is written,
+    which is what drives the mouth: the sprite is animated by the same audio
+    the speakers are getting, not by a guess about how long a word takes."""
     import sounddevice as sd
 
     with sd.OutputStream(
@@ -199,6 +245,11 @@ def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None
                 # now, not at the end of the buffered chunk.
                 out.abort()
                 return False
+            if on_level is not None:
+                try:
+                    on_level(level_of(chunk))
+                except Exception:
+                    on_level = None  # a broken listener must not stop playback
             out.write(chunk)
     return True
 
@@ -213,7 +264,8 @@ def speak_offline(text: str) -> None:
     engine.runAndWait()
 
 
-def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None) -> bool:
+def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None,
+          on_level=None) -> bool:
     """Speak *text*, preferring streaming ElevenLabs, then whole-clip
     ElevenLabs, then offline TTS. *on_error*, if given, is called with the
     exception when ElevenLabs fails (useful for logging) — a fallback still
@@ -233,13 +285,14 @@ def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] =
                 stream_pcm(text, voice_id=voice_id),
                 config.TTS_SAMPLE_RATE,
                 should_stop=should_stop,
+                on_level=on_level,
             )
         except TtsError as exc:
             if on_error is not None:
                 on_error(exc)
     try:
         pcm, sample_rate = synthesize_pcm(text, voice_id=voice_id)
-        return play_pcm(pcm, sample_rate, should_stop=should_stop)
+        return play_pcm(pcm, sample_rate, should_stop=should_stop, on_level=on_level)
     except TtsError as exc:
         if on_error is not None:
             on_error(exc)

bolt_pet/config.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import os
 import platform
+import re
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parent.parent  # project root (one above bolt_pet/)
@@ -29,6 +30,26 @@ def _load_env() -> None:
 _load_env()
 
 
+def set_env_value(key: str, value: str) -> None:
+    """Write *key=value* into .env, in place — replacing (or uncommenting)
+    an existing line for it, appending a new one otherwise. Used by the tray's
+    microphone picker so a device pick survives a restart without hand-editing
+    the file. Settings read once at import time (like MIC_DEVICE) still need
+    that restart to take effect; this only updates the file and os.environ."""
+    env_path = HERE / ".env"
+    lines = env_path.read_text(encoding="utf-8").splitlines() if env_path.exists() else []
+    pattern = re.compile(rf"^\s*#?\s*{re.escape(key)}\s*=")
+    new_line = f"{key}={value}"
+    for i, line in enumerate(lines):
+        if pattern.match(line):
+            lines[i] = new_line
+            break
+    else:
+        lines.append(new_line)
+    env_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    os.environ[key] = value
+
+
 def _node_name() -> str:
     try:
         return platform.node() or "desktop"
@@ -54,26 +75,41 @@ WAKE_WORD_THRESHOLD = float(os.environ.get("WAKE_WORD_THRESHOLD", "0.5"))
 # heartbeat poll in controller.py during quiet stretches with no wake word.
 WAKE_CHECK_INTERVAL_SECONDS = float(os.environ.get("WAKE_CHECK_INTERVAL_SECONDS", "1.2"))
 
-# ── STT (Deepgram, same as bolt_desk.py) ────────────────────────────────────
+# ── STT (server-hosted, same /desk/stt relay the Android app uses) ─────────
+# No local Deepgram account needed any more: audio/stt_stream.py opens a
+# websocket to this pet's own BOLT_SERVER_URL/DESK_API_KEY, which the server
+# relays to Deepgram and meters it against the same credit ledger as a chat turn.
+# There is no separate one-shot REST path server-side, so audio/stt.py's
+# "guaranteed" fallback uses this exact same connection too — just fed the
+# whole utterance at once instead of frame-by-frame.
 
-DEEPGRAM_API_KEY = os.environ.get("DEEPGRAM_API_KEY", "")
+# Transcribe *while* you talk instead of waiting for the utterance to end:
-DEEPGRAM_MODEL = os.environ.get("DEEPGRAM_MODEL", "nova-3")
+# frames go up as they are captured, so the transcript is ready the moment the
+# VAD says you stopped. Needs `websocket-client` (in requirements.txt); when
+# it can't connect the one-shot path (audio/stt.py) still tries the same
+# server relay itself, so turning this off only costs latency, not the
+# ability to transcribe at all.
+STT_STREAMING = os.environ.get("STT_STREAMING", "true").lower() in ("1", "true", "yes", "on")
 
-# ── TTS (ElevenLabs, requested as raw PCM so playback needs no external
+# ── TTS (server-hosted, same /desk/tts endpoint the Android app uses) ──────
-# player binary — cross-platform via sounddevice instead of shelling out to
+# No local ElevenLabs account needed for the normal reply voice any more —
-# mpv/ffplay like bolt_desk.py does on Linux) ───────────────────────────────
+# audio/tts.py posts to this pet's own BOLT_SERVER_URL/DESK_API_KEY and gets
+# back raw 16 kHz mono PCM16, same as the phone. Falls back to offline
+# pyttsx3 if the server call fails.
+#
+# ELEVENLABS_API_KEY is still read directly by this client for exactly one
+# feature the server has no endpoint for: multi-voice `dialoguectl` scenes
+# (audio/tts.synthesize_dialogue, see dialogue.py) — leave it blank and
+# everything except that one feature works with zero local API keys.
 
 ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY", "")
+# Which voice to ask the server for — an ElevenLabs voice id (or a "vb:"-
+# prefixed cloned voice, if this desk key owns one). The server picks the
+# synthesis model itself now; there is nothing left for this client to choose.
 ELEVENLABS_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID", "")
-ELEVENLABS_MODEL_ID = os.environ.get("ELEVENLABS_MODEL_ID", "eleven_flash_v2")
+# Fixed by the server (ai/desk_media.py's PCM_SAMPLE_RATE) — not a free
-# eleven_flash_v2 is English-only, and the two cases that swap the voice
+# tunable any more, but still an env override in case that ever changes.
-# (server-picked `speak_as`, or a reply with non-ASCII in it) are usually
+TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "16000"))
-# exactly the cases where the reply isn't English — see tts.model_for().
-ELEVENLABS_MULTILINGUAL_MODEL_ID = os.environ.get(
-    "ELEVENLABS_MULTILINGUAL_MODEL_ID", "eleven_flash_v2_5"
-)
-# ElevenLabs PCM output formats are named pcm_<sample_rate>.
-TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "24000"))
 
 # Does a voice the server picks (its speak_as marker — "talk like a pirate",
 # "say that in Japanese") stay on for later replies, or last one reply only?
@@ -100,7 +136,22 @@ DIALOGUE_VOICES = os.environ.get("DIALOGUE_VOICES", "")
 
 # ── mic / VAD (same tuning knobs as bolt_desk.py) ───────────────────────────
 
-MIC_DEVICE = os.environ.get("MIC_DEVICE", "") or None  # sounddevice name/index
+def _mic_device(raw: str):
+    """sounddevice treats an int index and a string name very differently: a
+    string is matched as a (fragile, often ambiguous) substring against every
+    device name, while an int is used as-is. .env only ever gives us a
+    string, so a numeric one — what the tray's mic picker saves, since the
+    same physical device usually shows up once per host API (MME,
+    DirectSound, WASAPI, ...) under the same name — has to be cast back to
+    int here, or it goes through the substring path and matches nothing (or
+    everything) instead."""
+    raw = raw.strip()
+    if not raw:
+        return None
+    return int(raw) if raw.lstrip("-").isdigit() else raw
+
+
+MIC_DEVICE = _mic_device(os.environ.get("MIC_DEVICE", ""))
 SPEAKER_DEVICE = os.environ.get("SPEAKER_DEVICE", "") or None
 
 RMS_THRESHOLD = int(os.environ.get("VAD_RMS_THRESHOLD", "300"))
@@ -168,6 +219,13 @@ BARGE_IN_FRAMES = int(os.environ.get("BARGE_IN_FRAMES", "4"))  # consecutive lou
 # waking the pet from idle (useful if Bolt's own voice trips the model).
 BARGE_IN_WAKE_THRESHOLD = float(os.environ.get("BARGE_IN_WAKE_THRESHOLD") or 0) or None
 
+# ── streaming the reply ─────────────────────────────────────────────────────
+# Speak each sentence as the server produces it, instead of waiting out the
+# whole model call before the first word. Falls back to the ordinary
+# request/response path automatically if the server has no streaming endpoint
+# or the stream fails before anything has been spoken.
+STREAMING_REPLIES = os.environ.get("STREAMING_REPLIES", "true").lower() in ("1", "true", "yes", "on")
+
 # ── streaming TTS ───────────────────────────────────────────────────────────
 # ElevenLabs' /stream endpoint + chunked playback: the pet starts talking
 # after the first PCM chunk instead of after the whole clip is synthesized.
@@ -307,6 +365,10 @@ PET_WANDER_MARGIN = int(os.environ.get("PET_WANDER_MARGIN", "20"))  # keep off s
 PET_SHAPED_INPUT = os.environ.get("PET_SHAPED_INPUT", "true").lower() in ("1", "true", "yes", "on")
 PET_CLICK_THROUGH = os.environ.get("PET_CLICK_THROUGH", "false").lower() in ("1", "true", "yes", "on")
 # Snap flush to a screen edge when dropped/parked within this many pixels of it.
+# Start where it was left rather than in the bottom-right corner. Ignored when
+# PET_START_X/Y pin it explicitly, and a saved position on a monitor that is no
+# longer plugged in is discarded rather than hiding the pet offscreen.
+PET_REMEMBER_POSITION = os.environ.get("PET_REMEMBER_POSITION", "true").lower() in ("1", "true", "yes", "on")
 PET_EDGE_SNAP = os.environ.get("PET_EDGE_SNAP", "true").lower() in ("1", "true", "yes", "on")
 PET_SNAP_MARGIN = int(os.environ.get("PET_SNAP_MARGIN", "48"))
 

bolt_pet/controller.py

@@ -20,15 +20,20 @@ from typing import Optional
 from PySide6.QtCore import QObject, Signal
 
 from . import (
-    config, dialogue as dialogue_mod, file_delivery, file_ops,
+    config, dialogue as dialogue_mod, speech, file_delivery, file_ops,
     history as history_mod, monitors as monitors_mod, notifications,
     pet_actions, quiet, screen_context, screen_text, self_restart,
     server_client, speech_text, updater,
 )
 from . import __version__
-from .audio import barge_in, mic, stt, tts, wake_word
+from .audio import barge_in, mic, stt, stt_stream, tts, wake_word
 from .state import PetState, PetStateMachine
 
+# A burst while the pet is busy is batched into one turn, but the queue still
+# needs a ceiling — a notification storm must not become an unbounded backlog
+# that gets read out minutes later.
+_MAX_PENDING_NOTIFICATIONS = 12
+
 # How often to re-check whether the pet should be napping. The fullscreen
 # probe shells out to xprop, so this deliberately isn't every heartbeat tick.
 _NAP_CHECK_INTERVAL_SECONDS = 10.0
@@ -41,6 +46,7 @@ class PetController(QObject):
     action = Signal(dict)  # parsed petctl action for the UI to perform
     napping = Signal(bool)  # quiet hours / fullscreen do-not-disturb
     voice_changed = Signal(str)  # name of the server-picked voice ("" = default)
+    mouth = Signal(float)  # 0..1 speech loudness, for lip-sync while talking
     restart_requested = Signal(str)  # version we just updated to
     finished = Signal()
 
@@ -77,6 +83,17 @@ class PetController(QObject):
         self._voice_name = ""
 
         self._barge_in: Optional[barge_in.BargeInDetector] = None
+        # Everything the pet says goes through here; see speech.py for why the
+        # four hand-rolled copies of this became one.
+        self._speaker = speech.Speaker(
+            state=self._state, tts=tts,
+            history=lambda text: self.history.add(history_mod.PET, text, time.time()),
+            on_said=self.said.emit, on_log=self.log.emit,
+            barge_in=lambda: self._barge_in,
+            detail_of=self._barge_in_detail,
+            voice_id=lambda: self._voice_id,
+            on_level=self.mouth.emit,
+        )
         self._napping = False
         self._nap_forced: Optional[bool] = None  # petctl nap on/off overrides the schedule
         self._last_nap_check = 0.0
@@ -249,21 +266,28 @@ class PetController(QObject):
             self._follow_ups = 0
 
         self._state.transition(PetState.LISTENING)
+        # Transcribe while they talk rather than after: frames go to Deepgram
+        # as they are captured, so the text is ready the moment the VAD says
+        # they stopped. None here just means the one-shot path will do it.
+        streamed = stt_stream.StreamingTranscriber.open()
         pcm = mic.record_utterance(
             self._stream,
             should_continue=self._should_continue,
             # Answering a question deserves longer than saying the wake word
             # on purpose does — you were just asked something.
             grace_s=config.FOLLOW_UP_GRACE_SECONDS if following_up else None,
+            on_frame=streamed.feed if streamed is not None else None,
         )
         if pcm is None:
+            if streamed is not None:
+                streamed.finish()
             self._follow_ups = 0  # silence ends the chain
             self._state.transition(PetState.IDLE)
             return
 
         self._state.transition(PetState.THINKING)
         try:
-            text = stt.transcribe(pcm)
+            text = self._transcribe(pcm, streamed)
         except stt.SttError as exc:
             self.log.emit(f"STT failed: {exc}")
             self._state.transition(PetState.ERROR)
@@ -278,9 +302,7 @@ class PetController(QObject):
         try:
             # What's focused right now rides along, so "what's this error?"
             # has a referent without you having to describe the window.
-            reply = server_client.converse(
+            reply = self._ask_server(self._with_context(text))
-                self._with_context(text), on_command=self._handle_command
-            )
         except server_client.ServerError as exc:
             self.log.emit(f"Server error: {exc}")
             self._state.transition(PetState.ERROR)
@@ -289,10 +311,34 @@ class PetController(QObject):
 
         self._check_deliveries()
         self._apply_voice(reply)
-        self._speak(reply.text)
+        if reply.spoken:
+            # Streamed: every sentence was spoken and logged as it arrived.
+            # The text still matters — a reply ending on a question should keep
+            # the mic open — but saying it again would repeat the whole answer.
+            self._after_speaking(reply.text, completed=True)
+        else:
+            self._speak(reply.text)
         self._state.transition(PetState.IDLE)
         self._maybe_self_restart()
 
+    def _transcribe(self, pcm, streamed) -> str:
+        """The transcript, from the live stream if it produced one.
+
+        The fallback is not a rare path to be tolerated — it is the safety net
+        that lets streaming be switched on at all. Whatever happened to the
+        socket, the full audio is still buffered here, so a failed stream costs
+        one ordinary upload and nothing else."""
+        if streamed is not None:
+            try:
+                text = streamed.finish()
+            except Exception:
+                self.log.emit("Streaming STT failed — falling back.")
+                text = ""
+            if text:
+                self.log.emit("(transcribed while you spoke)")
+                return text
+        return stt.transcribe(pcm)
+
     def _with_context(self, text: str) -> str:
         """Everything the server gets alongside what you actually said: the
         focused window title, and a one-line note about the screen layout so
@@ -492,6 +538,26 @@ class PetController(QObject):
             self._speak(reply.text)
         self._state.force(PetState.IDLE)
 
+    def _ask_server(self, text: str):
+        """One turn with the server, streamed when possible.
+
+        Streaming speaks each sentence as it is generated, so the wait is
+        time-to-first-sentence rather than the whole model call. It falls back
+        to the ordinary request/response path when the server has no streaming
+        endpoint, or when a stream dies *before* anything was spoken — after
+        that, retrying would say the first half twice."""
+        if config.STREAMING_REPLIES:
+            try:
+                return server_client.converse_stream(
+                    text, on_say=self._speak_stream_chunk,
+                    on_command=self._handle_command,
+                )
+            except server_client.ServerError as exc:
+                self.log.emit(f"Streaming unavailable ({exc}) — using the plain path.")
+        return server_client.converse(
+            text, on_command=self._handle_command, on_say=self._speak_holding,
+        )
+
     def _play_dialogue(self, scene: dict) -> str:
         """Play a `dialoguectl` scene and report back up the relay.
 
@@ -525,20 +591,8 @@ class PetController(QObject):
             # scene or fix the voice, and try again inside the same turn.
             return f"[dialogue] couldn't synthesize it: {exc}"
 
-        resume = self._state.state
+        completed = self._speaker.say_pcm(
-        self._state.transition(PetState.TALKING)
+            speech.Utterance.scene(text), pcm=pcm, sample_rate=sample_rate)
-        self.said.emit(speech_text.for_display(text))
-        self.history.add(history_mod.PET, text, time.time())
-
-        should_stop = None
-        if self._barge_in is not None:
-            self._barge_in.reset()
-            should_stop = self._barge_in.check
-        completed = tts.play_pcm(pcm, sample_rate, should_stop=should_stop)
-        if self._barge_in is not None:
-            self._barge_in.reset()  # the pet's own voices are in the wake window
-        if resume in (PetState.THINKING, PetState.IDLE):
-            self._state.transition(resume)
 
         if not completed:
             return dialogue_mod.describe(scene) + " (interrupted — they talked over it)"
@@ -569,32 +623,30 @@ class PetController(QObject):
             self.reset_voice()
 
     def _speak(self, text: str) -> None:
-        self._state.transition(PetState.TALKING)
+        """The answer: transcript, bubble, and the follow-up rule."""
-        # Bubble gets the markdown stripped but emoji kept (it can't render
+        completed = self._speaker.say(speech.Utterance.reply(text))
-        # **bold** but draws emoji fine); tts.speak() does its own, stricter
+        self._after_speaking(text, completed=completed,
-        # sanitizing for the voice.
+                             detail=self._speaker.last_detail)
-        self.said.emit(speech_text.for_display(text))
-        self.log.emit(f"Bolt: {text}")
-        self.history.add(history_mod.PET, text, time.time())
 
-        should_stop = None
+    def _speak_holding(self, text: str) -> None:
-        if self._barge_in is not None:
+        """"Give me a sec" while a tool runs — filler, so no transcript, and it
-            self._barge_in.reset()
+        resumes the state it interrupted because the turn isn't over."""
-            should_stop = self._barge_in.check
+        self._speaker.say(speech.Utterance.holding(text))
-        completed = tts.speak(
+
-            text,
+    def _speak_stream_chunk(self, text: str) -> None:
-            on_error=lambda exc: self.log.emit(f"TTS failed: {exc}"),
+        """One sentence of a streamed answer, spoken the moment it arrives."""
-            should_stop=should_stop,
+        completed = self._speaker.say(speech.Utterance.stream_chunk(text))
-            voice_id=self._voice_id or None,
+        if not completed:
-        )
+            self.log.emit("Interrupted — listening.")
-        # Read the scoring history *before* resetting, or the log reports the
+            self._follow_ups = 0
-        # blank counters instead of what actually fired.
+            self._talk_now.set()
-        detail = self._barge_in_detail()
+
-        if self._barge_in is not None:
+    def _after_speaking(self, text: str, *, completed: bool, detail: str = "") -> None:
-            # Playback fed the pet's own voice into the wake model's rolling
+        """What happens once an answer has been said, however it was said.
-            # window. Clear it before the idle listener starts scoring again,
+
-            # or Bolt's last sentence is still in there being re-scored.
+        Shared by the plain and streamed paths: a streamed reply is spoken
-            self._barge_in.reset()
+        sentence by sentence, but it still has to obey the same rules about
+        keeping the mic open when it ended on a question."""
         if not completed:
             # You talked over it — take that as the start of the next turn
             # rather than making you say the wake word again.
@@ -614,20 +666,22 @@ class PetController(QObject):
     def _should_follow_up(self, text: str) -> bool:
         """Whether *text* leaves the pet waiting on an answer.
 
-        Muted is excluded because mute means "don't listen to me" — an
+        The rule itself — question, cap, off switch — is
-        automatic turn would walk straight past it. Napping isn't: quiet
+        `speech.follow_up_decision`, because it is a rule with an off-by-one in
-        hours suppress the pet *starting* something, and a question is only
+        it and deserves a test that needs no audio. What stays here is the part
-        ever asked in reply to you."""
+        that is genuinely the controller's: mute, and the log line. Muted is
-        if not config.FOLLOW_UP_LISTEN or self._muted:
+        excluded because mute means "don't listen to me" and an automatic turn
+        would walk straight past it. Napping isn't: quiet hours suppress the
+        pet *starting* something, and a question is only ever asked in reply to
+        you."""
+        if self._muted:
             return False
-        if not speech_text.is_question(text):
+        keep, why = speech.follow_up_decision(text, completed=True, follow_ups=self._follow_ups)
-            return False
+        if not keep and "cap" in why:
-        # Only worth mentioning the cap on a reply that would otherwise have
+            # Only worth mentioning the cap on a reply that would otherwise have
-        # kept listening, or it fires on every statement the pet makes.
+            # kept listening, or it fires on every statement the pet makes.
-        if config.FOLLOW_UP_MAX_TURNS > 0 and self._follow_ups >= config.FOLLOW_UP_MAX_TURNS:
             self.log.emit("Follow-up limit reached — say the wake word to keep going.")
-            return False
+        return keep
-        return True
 
     def _barge_in_detail(self) -> str:
         """Why the interruption fired, for the log. How far into playback it
@@ -685,33 +739,54 @@ class PetController(QObject):
 
     def _queue_notification(self, notification: notifications.Notification) -> None:
         """Called on the watcher thread — just queue it; forwarding happens on
-        the pipeline thread where it can't collide with a live conversation."""
+        the pipeline thread where it can't collide with a live conversation.
-        if not self._notification_gate.should_forward(notification, time.monotonic()):
+
+        Only the *filter* applies here. The rate limit used to as well, which
+        meant a second message arriving inside the window was silently thrown
+        away — with NOTIFICATION_MIN_INTERVAL_SECONDS=60, two texts a minute
+        apart and you only ever heard about one of them. Losing a message from
+        a person to save a round trip is the wrong trade; they are batched at
+        the far end instead, which costs the same one round trip and keeps
+        them all."""
+        if not self._notification_gate.matches(notification):
             return
         with self._notification_lock:
+            if len(self._pending_notifications) >= _MAX_PENDING_NOTIFICATIONS:
+                self._pending_notifications.pop(0)  # bound it; oldest goes first
             self._pending_notifications.append(notification)
 
     def _drain_notifications(self) -> None:
+        """Forward everything waiting as ONE turn.
+
+        Batching is what makes it safe to keep every notification: five that
+        arrived while the pet was mid-conversation become one message and one
+        round trip, instead of five separate interruptions queued up to fire
+        back to back."""
         with self._notification_lock:
             pending, self._pending_notifications = self._pending_notifications, []
+        if not pending or not self._running or self._napping:
+            return
         for notification in pending:
-            if not self._running or self._napping:
-                return
             self.log.emit(f"Notification: {notification.as_text()}")
             self.history.add(history_mod.SYSTEM, notification.as_text(), time.time())
-            try:
+
-                reply = server_client.converse(
+        if len(pending) == 1:
-                    f"[desktop notification] {notification.as_text()}",
+            message = f"[desktop notification] {pending[0].as_text()}"
-                    on_command=self._handle_command,
+        else:
-                )
+            lines = "\n".join(f"- {n.as_text()}" for n in pending)
-            except server_client.ServerError as exc:
+            message = f"[{len(pending)} desktop notifications]\n{lines}"
-                self.log.emit(f"Couldn't forward notification: {exc}")
+        try:
-                return
+            reply = server_client.converse(
-            self._check_deliveries()
+                message, on_command=self._handle_command, on_say=self._speak_holding,
-            self._apply_voice(reply)
+            )
-            if reply.text.strip():
+        except server_client.ServerError as exc:
-                self._speak(reply.text)
+            self.log.emit(f"Couldn't forward notification: {exc}")
-                self._state.transition(PetState.IDLE)
+            return
+        self._check_deliveries()
+        self._apply_voice(reply)
+        if reply.text.strip() and not reply.spoken:
+            self._speak(reply.text)
+        self._state.transition(PetState.IDLE)
 
     # ── file delivery ────────────────────────────────────────────────────
 
@@ -784,20 +859,36 @@ class PetController(QObject):
     # ── heartbeat ────────────────────────────────────────────────────────
 
     def _maybe_heartbeat(self) -> None:
+        """Called from the wake listener's tick (~every WAKE_CHECK_INTERVAL_SECONDS).
+
+        Two different cadences live here, and conflating them was costing
+        minutes. A *notification* is an event that already happened — it should
+        go out as soon as the pet is free, which is the next tick. The
+        *heartbeat* is a poll, and polling the server every 1.2s would be
+        absurd, so it stays on its own interval."""
         self._refresh_nap_state()
         self._maybe_update()
         if self._update_pending:
             return  # on the way out — don't start a conversation now
+
+        # Notifications: every tick, not every heartbeat.
+        if self._state.state == PetState.IDLE and not self._napping:
+            self._drain_notifications()
+
         now = time.monotonic()
         if now - self._last_heartbeat < config.HEARTBEAT_INTERVAL_SECONDS:
             return
-        self._last_heartbeat = now
         if self._state.state != PetState.IDLE:
+            # Mid-conversation. Do NOT stamp the clock — an earlier version
+            # did, so a heartbeat that landed while the pet was talking burned
+            # its slot and waited another full interval. With follow-up
+            # listening that could repeat for several cycles, which is why a
+            # notification could sit unspoken for five or ten minutes.
             return
         if self._napping:
             return  # quiet hours: still answers when spoken to, just doesn't start
+        self._last_heartbeat = now
         self._check_deliveries()
-        self._drain_notifications()
         if self._state.state != PetState.IDLE:
             return
         try:

bolt_pet/dialogue.py

@@ -224,4 +224,12 @@ def describe(action: dict, *, played: bool = True) -> str:
     return (
         f"[dialogue] played {len(lines)} line{'s' if len(lines) != 1 else ''} "
         f"in {len(voices)} voice{'s' if len(voices) != 1 else ''}: {', '.join(voices)}{note}"
+        # The scene was spoken out loud before this result got back to the
+        # server, and the model has no other way to know that. Without saying
+        # so it writes a final reply summarising what the user just heard, and
+        # the pet says the same thing twice in a row (observed 2026-07-31).
+        # An instruction delivered here, at the moment it applies, lands far
+        # better than a rule buried in a long system prompt.
+        "\nThe user HEARD this already. Do not repeat, summarise or narrate it "
+        "in your reply — answer with at most one short line, or nothing new."
     )

bolt_pet/doctor.py

@@ -0,0 +1,323 @@
+"""`python -m bolt_pet --doctor` — is this install actually going to work?
+
+Written after a week of debugging things a preflight would have shown in one
+command: a missing port publish, a wrong reverse-proxy header, a venv whose
+python was a zero-byte file, an OCR engine that was never installed. Every one
+of those presented as "the pet is being weird" and took a conversation to find.
+
+Each check is independent and reports one of three things — ok, a warning
+(works, but degraded), or a failure (this will not do what you expect) — and
+says *what to do about it* rather than just what is wrong. Nothing here raises:
+a doctor that crashes on a broken install is diagnosing the wrong patient.
+
+Deliberately does not open the mic or call a paid API by default. It checks
+that the door is unlocked, not that the room is furnished; `--deep` is there
+when you want it to actually knock.
+"""
+
+from __future__ import annotations
+
+import importlib
+import shutil
+import sys
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from . import config
+
+OK, WARN, FAIL = "ok", "warn", "fail"
+
+_MARKS = {OK: "  ok  ", WARN: " warn ", FAIL: " FAIL "}
+
+
+@dataclass
+class Check:
+    name: str
+    status: str
+    detail: str = ""
+    fix: str = ""
+
+    def line(self) -> str:
+        text = f"[{_MARKS[self.status]}] {self.name:22} {self.detail}"
+        if self.fix and self.status != OK:
+            text += f"\n{'':32}→ {self.fix}"
+        return text
+
+
+def _module(name: str) -> bool:
+    try:
+        importlib.import_module(name)
+        return True
+    except Exception:
+        return False
+
+
+# ── the checks ──────────────────────────────────────────────────────────────
+
+def check_config() -> Check:
+    missing = config.missing_config()
+    if missing:
+        return Check("server config", FAIL, f"missing {', '.join(missing)}",
+                     "set them in .env — without these the controller exits at startup")
+    return Check("server config", OK, f"{config.SERVER_URL} as {config.SESSION_ID}")
+
+
+def check_server(deep: bool = False) -> Check:
+    if not config.SERVER_URL:
+        return Check("server", FAIL, "no BOLT_SERVER_URL",
+                     "cp .env.example .env and set BOLT_SERVER_URL to your Bolt server")
+    if not deep:
+        return Check("server", OK, f"{config.SERVER_URL} (not contacted; --deep to try)")
+    try:
+        from . import server_client
+
+        health = server_client.check_health(timeout=8)
+        return Check("server", OK, f"reachable — {health}")
+    except Exception as exc:
+        return Check("server", FAIL, f"unreachable: {exc}",
+                     "check the URL, the key, and that the container is up")
+
+
+def check_streaming_endpoint(deep: bool = False) -> Check:
+    """The streamed-reply endpoint is newer than some deployed servers."""
+    if not config.STREAMING_REPLIES:
+        return Check("streamed replies", WARN, "disabled (STREAMING_REPLIES=false)")
+    if not deep:
+        return Check("streamed replies", OK, "enabled (endpoint not probed)")
+    try:
+        import requests
+
+        response = requests.post(
+            f"{config.SERVER_URL}/desk/converse_stream",
+            json={"session_id": config.SESSION_ID, "text": ""},
+            headers={"X-Desk-Api-Key": config.API_KEY}, timeout=8, stream=True,
+        )
+        if response.status_code == 404:
+            return Check("streamed replies", WARN, "server has no /desk/converse_stream",
+                         "update the server, or set STREAMING_REPLIES=false to skip the probe")
+        return Check("streamed replies", OK, f"endpoint answered {response.status_code}")
+    except Exception as exc:
+        return Check("streamed replies", WARN, f"probe failed: {exc}")
+
+
+def check_microphone(deep: bool = False) -> Check:
+    if not _module("sounddevice"):
+        return Check("microphone", FAIL, "sounddevice is not installed",
+                     "pip install -r requirements.txt")
+    try:
+        import sounddevice as sd
+
+        devices = [d for d in sd.query_devices() if d.get("max_input_channels", 0) > 0]
+        if not devices:
+            return Check("microphone", FAIL, "no input devices",
+                         "on Linux check PipeWire/PulseAudio is running as your user")
+        chosen = config.MIC_DEVICE or "system default"
+        if not deep:
+            return Check("microphone", OK, f"{len(devices)} input device(s), using {chosen}")
+        with sd.InputStream(samplerate=config.SAMPLE_RATE, channels=1, dtype="int16",
+                            device=config.MIC_DEVICE, blocksize=config.FRAME_LEN):
+            pass
+        return Check("microphone", OK, f"opened at {config.SAMPLE_RATE} Hz ({chosen})")
+    except Exception as exc:
+        return Check("microphone", FAIL, f"could not open: {exc}",
+                     "don't run the pet as root — PortAudio can't reach your PipeWire socket")
+
+
+def check_wake_model() -> Check:
+    from pathlib import Path
+
+    path = Path(config.WAKE_MODEL_PATH)
+    if not path.exists():
+        return Check("wake word", FAIL, f"{path.name} is missing",
+                     "it ships in the project root; check WAKE_MODEL_FILE")
+    if not _module("openwakeword"):
+        return Check("wake word", FAIL, "openwakeword is not installed",
+                     "pip install -r requirements.txt")
+    return Check("wake word", OK, f"{path.name}, threshold {config.WAKE_WORD_THRESHOLD}")
+
+
+def check_stt() -> Check:
+    """STT is a websocket relay to the server (/desk/stt) — no local Deepgram
+    account, but websocket-client is now load-bearing for transcription to
+    work at all, not just the streaming optimisation (there's no separate
+    REST fallback any more)."""
+    if not config.is_configured():
+        return Check("speech-to-text", FAIL, "no BOLT_SERVER_URL/DESK_API_KEY",
+                     "set them in .env — nothing you say can be transcribed without the server")
+    if not _module("websocket"):
+        return Check("speech-to-text", FAIL, "websocket-client is missing",
+                     "pip install -r requirements.txt — /desk/stt is a websocket relay "
+                     "with no REST fallback")
+    mode = "streaming" if config.STT_STREAMING else "one-shot (still via the server relay)"
+    return Check("speech-to-text", OK, f"server relay, {mode}")
+
+
+def check_tts(deep: bool = False) -> Check:
+    """TTS is the server's /desk/tts — no local ElevenLabs account needed for
+    the normal reply voice, just a voice id for it to request.
+
+    Shallow only checks config presence, same as the other shallow checks —
+    it can't see a live failure (bad voice id, an expired key, a Voicebox
+    clone owned by someone else). --deep actually calls /desk/tts with a
+    couple of words and reports the real HTTP status/error, which is the
+    only way to tell "misconfigured" apart from "configured but broken"
+    (e.g. speak() silently falling back to the offline voice)."""
+    if not (config.is_configured() and config.ELEVENLABS_VOICE_ID):
+        if _module("pyttsx3"):
+            return Check("text-to-speech", WARN, "server/voice not configured — offline voice only",
+                         "set BOLT_SERVER_URL/DESK_API_KEY and ELEVENLABS_VOICE_ID for the real voice")
+        return Check("text-to-speech", FAIL, "no server/voice config and no pyttsx3 fallback",
+                     "set BOLT_SERVER_URL/DESK_API_KEY/ELEVENLABS_VOICE_ID, "
+                     "or pip install pyttsx3 for an offline voice")
+    if not deep:
+        return Check("text-to-speech", OK,
+                     f"server relay, voice …{config.ELEVENLABS_VOICE_ID[-6:]} "
+                     "(not probed; --deep to actually synthesize)")
+    try:
+        import requests
+
+        response = requests.post(
+            f"{config.SERVER_URL}/desk/tts",
+            headers={"X-Desk-Api-Key": config.API_KEY},
+            json={"session_id": config.SESSION_ID, "text": "testing",
+                  "voice_id": config.ELEVENLABS_VOICE_ID},
+            timeout=20,
+        )
+    except Exception as exc:
+        return Check("text-to-speech", FAIL, f"probe failed: {exc}",
+                     "check BOLT_SERVER_URL is reachable from this machine")
+    if response.status_code == 200 and response.content:
+        return Check("text-to-speech", OK,
+                     f"server synthesized {len(response.content)} bytes "
+                     f"for voice …{config.ELEVENLABS_VOICE_ID[-6:]}")
+    detail = ""
+    try:
+        detail = str(response.json().get("error") or "")
+    except Exception:
+        pass
+    return Check(
+        "text-to-speech", FAIL,
+        f"server returned {response.status_code}{': ' + detail if detail else ''} "
+        f"for voice …{config.ELEVENLABS_VOICE_ID[-6:]}",
+        "a 404 means this voice id isn't yours (or doesn't exist); 402 means "
+        "the account is out of credits; 401 means DESK_API_KEY is wrong",
+    )
+
+
+def check_dialogue() -> Check:
+    if not config.DIALOGUE:
+        return Check("multi-voice scenes", WARN, "disabled (DIALOGUE=false)")
+    if not config.ELEVENLABS_API_KEY:
+        return Check("multi-voice scenes", WARN, "no ELEVENLABS_API_KEY",
+                     "set it in .env — dialogue scenes are the one feature still calling "
+                     "ElevenLabs directly, since the server has no equivalent endpoint")
+    from . import dialogue
+
+    cast = dialogue.parse_voice_map(config.DIALOGUE_VOICES)
+    if not cast:
+        return Check("multi-voice scenes", WARN, "no cast configured — only 'self' works",
+                     'set DIALOGUE_VOICES=narrator:<id>,villain:<id>')
+    return Check("multi-voice scenes", OK, f"{len(cast)} voice(s): {', '.join(sorted(cast))}")
+
+
+def check_screen_text() -> Check:
+    if not config.SCREEN_TEXT:
+        return Check("screen reading", WARN, "disabled (SCREEN_TEXT=false)")
+    if not _module("mss"):
+        return Check("screen reading", WARN, "mss is not installed — petctl read will decline",
+                     "pip install mss (and note it cannot capture on Wayland)")
+    engine = "pytesseract" if _module("pytesseract") else (
+        "rapidocr" if _module("rapidocr_onnxruntime") else "")
+    if not engine:
+        return Check("screen reading", WARN, "no OCR engine",
+                     "pip install pytesseract && apt install tesseract-ocr, "
+                     "or pip install rapidocr-onnxruntime")
+    if engine == "pytesseract" and not shutil.which("tesseract"):
+        return Check("screen reading", WARN, "pytesseract is installed but tesseract is not",
+                     "apt install tesseract-ocr")
+    return Check("screen reading", OK, f"mss + {engine}")
+
+
+def check_hotkey() -> Check:
+    if not _module("pynput"):
+        return Check("push-to-talk", WARN, "pynput is not installed",
+                     "the wake word still works; pip install pynput for the hotkey")
+    import os
+
+    if os.environ.get("WAYLAND_DISPLAY") and not os.environ.get("DISPLAY"):
+        return Check("push-to-talk", WARN, "Wayland session — global hotkeys usually blocked",
+                     "use the wake word, or click the pet")
+    return Check("push-to-talk", OK, config.PUSH_TO_TALK_HOTKEY)
+
+
+def check_sprites() -> Check:
+    from pathlib import Path
+
+    root = Path(__file__).resolve().parent / "assets" / "sprites"
+    if not root.exists():
+        return Check("sprites", WARN, "no art — the placeholder blob will be drawn",
+                     "python scripts/generate_bolt_sprites.py")
+    counts = {d.name: len(list(d.glob("*.png"))) for d in sorted(root.iterdir()) if d.is_dir()}
+    empty = [name for name, count in counts.items() if count == 0]
+    if empty:
+        return Check("sprites", WARN, f"no frames for: {', '.join(empty)}",
+                     "python scripts/generate_bolt_sprites.py")
+    return Check("sprites", OK, ", ".join(f"{n} {c}" for n, c in counts.items()))
+
+
+def check_latency() -> Check:
+    """The setting most likely to make it feel slow, and the least obvious."""
+    silence = config.SILENCE_END_SEC
+    if silence >= 1.5:
+        return Check("turn latency", WARN, f"VAD_SILENCE_END_SEC={silence:g}s of dead air per turn",
+                     "0.8-1.0 feels markedly snappier; it is pure wait before anything starts")
+    return Check("turn latency", OK, f"silence timeout {silence:g}s, "
+                 f"streaming {'on' if config.STREAMING_REPLIES else 'off'}")
+
+
+CHECKS: tuple[tuple[str, Callable], ...] = (
+    ("config", check_config),
+    ("server", check_server),
+    ("stream", check_streaming_endpoint),
+    ("mic", check_microphone),
+    ("wake", check_wake_model),
+    ("stt", check_stt),
+    ("tts", check_tts),
+    ("dialogue", check_dialogue),
+    ("screen", check_screen_text),
+    ("hotkey", check_hotkey),
+    ("sprites", check_sprites),
+    ("latency", check_latency),
+)
+
+
+def run(deep: bool = False) -> list[Check]:
+    results = []
+    for _name, check in CHECKS:
+        try:
+            try:
+                results.append(check(deep))
+            except TypeError:
+                results.append(check())
+        except Exception as exc:  # a broken check must not hide the others
+            results.append(Check(_name, FAIL, f"the check itself failed: {exc}"))
+    return results
+
+
+def main(argv: Optional[list] = None) -> int:
+    argv = list(argv if argv is not None else sys.argv[1:])
+    deep = "--deep" in argv
+    print(f"Bolt pet preflight{' (deep: contacting the server and opening the mic)' if deep else ''}\n")
+    results = run(deep=deep)
+    for check in results:
+        print(check.line())
+    failures = [c for c in results if c.status == FAIL]
+    warnings = [c for c in results if c.status == WARN]
+    print()
+    if failures:
+        print(f"{len(failures)} problem(s) will stop this working. Fix those first.")
+    elif warnings:
+        print(f"Ready. {len(warnings)} thing(s) degraded but working.")
+    else:
+        print("Everything checks out.")
+    return 1 if failures else 0

bolt_pet/server_client.py

@@ -19,6 +19,7 @@ Kept dependency-free beyond `requests` so it's easy to unit test with mocks.
 
 from __future__ import annotations
 
+import json
 import subprocess
 from pathlib import Path
 from typing import Callable, NamedTuple, Optional
@@ -43,6 +44,10 @@ class Reply(NamedTuple):
     text: str
     voice_id: str = ""
     voice_name: str = ""
+    # True when the sentences were already spoken as they streamed in. The
+    # text is still carried — the follow-up rule needs to see whether the
+    # answer ended on a question — it just must not be read out again.
+    spoken: bool = False
 
 
 def _headers() -> dict:
@@ -90,12 +95,20 @@ def converse(
     text: str,
     on_command: Callable[[str], str] = run_local_command,
     timeout: float = 120.0,
+    on_say: Optional[Callable[[str], None]] = None,
 ) -> Reply:
     """Send one turn of conversation to the desk API, relaying any commands
     the server sends back until it produces a final reply.
 
     *on_command* is injectable for tests; defaults to actually running the
     command locally (matching bolt_desk.py's behavior).
+
+    *on_say* is called with a short holding line ("give me a sec") when the
+    server sends one alongside a command. It is the difference between silence
+    and an answer while a tool runs: the model's acknowledgement used to be
+    discarded server-side, so the whole round trip was dead air and the model
+    then repeated itself in the final reply. Optional, so an older pet against
+    a newer server simply stays quiet as before.
     """
     headers = _headers()
     try:
@@ -111,6 +124,13 @@ def converse(
     for _ in range(_MAX_RELAY_HOPS):
         if payload.get("type") != "command":
             break
+        holding = str(payload.get("say") or "").strip()
+        if holding and on_say is not None:
+            # Spoken *before* the command runs — that is the whole point.
+            try:
+                on_say(holding)
+            except Exception:
+                pass  # a failed acknowledgement must not cost the tool call
         output = on_command(str(payload.get("command") or ""))
         try:
             response = requests.post(
@@ -135,6 +155,104 @@ def converse(
     raise ServerError(str(payload.get("error") or "unknown server response"))
 
 
+def converse_stream(
+    text: str,
+    on_say: Callable[[str], None],
+    on_command: Callable[[str], str] = run_local_command,
+    timeout: float = 180.0,
+) -> Reply:
+    """Same turn as converse(), but speaking each sentence as it arrives.
+
+    Without this the pet waits out the *entire* model call before a single
+    word is heard; with it the wait is time-to-first-sentence, which on a
+    multi-sentence answer is most of the difference.
+
+    Falls back by raising ServerError before anything has been spoken — the
+    caller then retries the ordinary path and the user never finds out. Once a
+    sentence *has* been spoken there is no going back, so late failures end the
+    turn with whatever was said rather than repeating it.
+    """
+    spoke_anything = False
+    try:
+        response = requests.post(
+            f"{config.SERVER_URL}/desk/converse_stream",
+            json={"session_id": config.SESSION_ID, "text": text},
+            headers=_headers(), timeout=timeout, stream=True,
+        )
+        response.raise_for_status()
+        for raw in response.iter_lines(decode_unicode=True):
+            if not raw:
+                continue
+            try:
+                event = json.loads(raw)
+            except (TypeError, ValueError):
+                continue
+            kind = str(event.get("type") or "")
+
+            if kind == "say":
+                line = str(event.get("text") or "").strip()
+                if line:
+                    spoke_anything = True
+                    on_say(line)
+            elif kind == "command":
+                # The tool loop is request/response, so the rest of the turn
+                # finishes through the ordinary relay rather than inside the
+                # stream — one protocol for tools, not two.
+                response.close()
+                return _finish_relay(event, on_command, on_say)
+            elif kind == "reply":
+                return Reply(
+                    text=str(event.get("text") or ""),
+                    voice_id=str(event.get("voice_id") or ""),
+                    voice_name=str(event.get("voice_name") or ""),
+                    spoken=bool(event.get("already_spoken")),
+                )
+            elif kind == "error":
+                raise ServerError(str(event.get("error") or "stream failed"))
+    except ServerError:
+        raise
+    except Exception as exc:
+        if spoke_anything:
+            # Half a reply is out loud already; ending quietly beats saying it
+            # all again through the fallback path.
+            return Reply(text="", spoken=True)
+        raise ServerError(f"streaming failed: {exc}") from exc
+    raise ServerError("stream ended without a reply")
+
+
+def _finish_relay(event: dict, on_command, on_say) -> Reply:
+    """Run the tool the stream handed over, then continue the classic relay."""
+    payload = dict(event)
+    headers = _headers()
+    for _ in range(_MAX_RELAY_HOPS):
+        if payload.get("type") != "command":
+            break
+        holding = str(payload.get("say") or "").strip()
+        if holding and on_say is not None:
+            try:
+                on_say(holding)
+            except Exception:
+                pass
+        output = on_command(str(payload.get("command") or ""))
+        try:
+            response = requests.post(
+                f"{config.SERVER_URL}/desk/tool_result",
+                json={"session_id": config.SESSION_ID,
+                      "token": payload.get("token"), "output": output},
+                headers=headers, timeout=180,
+            )
+            payload = response.json()
+        except Exception as exc:
+            raise ServerError(f"couldn't reach the server during tool relay: {exc}") from exc
+    if payload.get("type") == "reply":
+        return Reply(
+            text=str(payload.get("text") or ""),
+            voice_id=str(payload.get("voice_id") or ""),
+            voice_name=str(payload.get("voice_name") or ""),
+        )
+    raise ServerError(str(payload.get("error") or "unknown server response"))
+
+
 def list_outbox_files(timeout: float = 15.0) -> list:
     """Files the server has queued for this session via its deliver_files
     tool (e.g. "send me that report" during a conversation) — each entry has

bolt_pet/speech.py

@@ -0,0 +1,194 @@
+"""Everything the pet says, and the policy differences between kinds of saying.
+
+There used to be four of these in controller.py — a reply, a holding line, a
+streamed sentence, a dialogue scene — each written when its feature was built,
+each repeating the same dance: transition state, show the bubble, maybe record
+history, reset barge-in, call TTS, reset barge-in again, resume state, decide
+whether to keep the mic open. Only the *policy* differed, and the copies had
+already started to drift: one forgot to arm barge-in, another logged a
+different prefix, a third skipped the follow-up rule.
+
+So the dance lives here once, and the differences are data:
+
+    reply        the answer. Transcript, bubble, follow-up rule, ends IDLE.
+    holding      "give me a sec" while a tool runs. No transcript — it is
+                 filler, and the transcript should keep the answer. Resumes
+                 whatever state it interrupted, because the turn isn't over.
+    stream       one sentence of a streamed answer. Transcript and bubble like
+                 a reply, but stays TALKING so the sprite doesn't flicker
+                 between sentences, and the follow-up rule waits for the last.
+    scene        a dialoguectl take. Transcript (the user heard it), resumes
+                 mid-turn like a holding line.
+
+The controller keeps the pipeline; this keeps the rules about talking.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from . import config, speech_text
+from .state import PetState
+
+
+@dataclass(frozen=True)
+class Utterance:
+    """One thing to say, and how saying it should behave."""
+
+    text: str
+    record: bool = True         # goes in the transcript, or is it filler?
+    resume: bool = False        # return to the state it interrupted (mid-turn)
+    hold_talking: bool = False  # stay TALKING afterwards (more is coming)
+    follow_up: bool = True      # may leave the mic open if it ends on a question
+    interruptible: bool = True  # arm barge-in for this one
+    log_prefix: str = "Bolt"
+
+    @classmethod
+    def reply(cls, text: str) -> "Utterance":
+        return cls(text)
+
+    @classmethod
+    def holding(cls, text: str) -> "Utterance":
+        # Filler: no transcript, no follow-up, and not interruptible — cutting
+        # off "give me a sec" would strand the tool that is already running.
+        return cls(text, record=False, resume=True, follow_up=False,
+                   interruptible=False, log_prefix="Bolt (holding)")
+
+    @classmethod
+    def stream_chunk(cls, text: str) -> "Utterance":
+        return cls(text, hold_talking=True, follow_up=False)
+
+    @classmethod
+    def scene(cls, text: str) -> "Utterance":
+        return cls(text, resume=True, follow_up=False)
+
+
+class Speaker:
+    """Says things on behalf of the controller.
+
+    Collaborators are passed in rather than reached for, so the whole of
+    speaking is testable without Qt, audio hardware or a state machine: hand it
+    fakes and assert on what came out."""
+
+    def __init__(self, *, state, tts, history=None, on_said=None, on_log=None,
+                 barge_in: Callable[[], object] = lambda: None,
+                 voice_id: Callable[[], str] = lambda: "",
+                 detail_of: Callable[[], str] = lambda: "",
+                 on_level: Optional[Callable[[float], None]] = None):
+        self._state = state
+        self._tts = tts
+        self._history = history
+        self._on_said = on_said or (lambda _text: None)
+        self._on_log = on_log or (lambda _msg: None)
+        # Read through a callable rather than held: the detector is built after
+        # the speaker (it needs the mic stream), replaced when barge-in mode
+        # changes, and swapped by tests. Two copies of it drifting apart is a
+        # bug nobody notices until the wake model starts hearing the pet.
+        self._barge_in_of = barge_in
+        self._voice_id = voice_id
+        # What actually fired, captured BEFORE the post-playback reset. Read it
+        # afterwards and every interruption reports 0.000 at frame 0 — which
+        # looks like hard evidence and is nothing of the sort.
+        self._detail_of = detail_of
+        self.last_detail = ""
+        self._on_level = on_level
+
+    @property
+    def _barge_in(self):
+        try:
+            return self._barge_in_of()
+        except Exception:
+            return None
+
+    def say(self, utterance: Utterance) -> bool:
+        """Speak it. Returns False if it was interrupted.
+
+        The one place that knows the order these steps go in — which is the
+        point, because getting that order wrong is invisible until the wake
+        model starts hearing the pet's own voice."""
+        line = speech_text.for_display(utterance.text)
+        if not line:
+            return True
+
+        resume_state = self._state.state
+        if self._state.state != PetState.TALKING:
+            self._state.transition(PetState.TALKING)
+        self._on_said(line)
+        self._on_log(f"{utterance.log_prefix}: {line}")
+        if utterance.record and self._history is not None:
+            self._history(utterance.text)
+
+        should_stop = None
+        if self._barge_in is not None and utterance.interruptible:
+            self._barge_in.reset()
+            should_stop = self._barge_in.check
+
+        completed = self._tts.speak(
+            utterance.text,
+            on_error=lambda exc: self._on_log(f"TTS failed: {exc}"),
+            should_stop=should_stop,
+            voice_id=self._voice_id() or None,
+            on_level=self._on_level,
+        )
+        self.last_detail = self._detail_of()
+        if self._barge_in is not None:
+            # Playback fed the pet's own voice into the wake model's rolling
+            # window. Clear it before the idle listener scores again, or the
+            # last sentence is still in there being re-heard.
+            self._barge_in.reset()
+        if self._on_level is not None:
+            self._on_level(0.0)  # mouth closed; nothing is playing now
+
+        if utterance.resume and resume_state in (PetState.THINKING, PetState.IDLE):
+            self._state.transition(resume_state)
+        return bool(completed)
+
+
+    def say_pcm(self, utterance: Utterance, *, pcm, sample_rate: int) -> bool:
+        """Speak audio that is already synthesized — a dialoguectl scene.
+
+        Same policy, same barge-in handling, same mouth; only the source of
+        the samples differs. Sharing this is why a scene can be talked over
+        exactly like an ordinary reply."""
+        line = speech_text.for_display(utterance.text)
+        resume_state = self._state.state
+        if self._state.state != PetState.TALKING:
+            self._state.transition(PetState.TALKING)
+        if line:
+            self._on_said(line)
+        self._on_log(f"{utterance.log_prefix}: {line}")
+        if utterance.record and self._history is not None:
+            self._history(utterance.text)
+
+        should_stop = None
+        if self._barge_in is not None and utterance.interruptible:
+            self._barge_in.reset()
+            should_stop = self._barge_in.check
+        completed = self._tts.play_pcm(
+            pcm, sample_rate, should_stop=should_stop, on_level=self._on_level)
+        self.last_detail = self._detail_of()
+        if self._barge_in is not None:
+            self._barge_in.reset()
+        if self._on_level is not None:
+            self._on_level(0.0)
+        if utterance.resume and resume_state in (PetState.THINKING, PetState.IDLE):
+            self._state.transition(resume_state)
+        return bool(completed)
+
+
+def follow_up_decision(text: str, *, completed: bool, follow_ups: int) -> tuple[bool, str]:
+    """Whether to keep listening after speaking, and why.
+
+    Split out as a pure function because it is a *rule* with an off-by-one cap
+    in it, and rules with counters deserve a test that doesn't need audio."""
+    if not completed:
+        return True, "interrupted"
+    if not speech_text.is_question(text):
+        return False, ""
+    cap = config.FOLLOW_UP_MAX_TURNS
+    if not config.FOLLOW_UP_LISTEN:
+        return False, ""
+    if cap > 0 and follow_ups >= cap:
+        return False, f"follow-up cap ({cap}) reached"
+    return True, "question"

bolt_pet/speech_text.py

@@ -97,10 +97,34 @@ def _bullets_to_sentences(text: str) -> str:
     return " ".join(line if line[-1] in ".!?:,;" else line + "." for line in lines)
 
 
+# ElevenLabs v3 delivery tags — "[laughing]", "[whispering]", "[sighs]". They
+# are instructions to the *dialogue* model (see dialogue.py), and only inside a
+# dialoguectl scene. Left in an ordinary reply they reach eleven_flash_v2,
+# which has no idea what they mean and simply reads the word: observed live
+# 2026-07-31, the pet announcing "laughing That one came through clean".
+#
+# Matched narrowly — a bracketed adverb/gerund, or one of the noise words that
+# aren't either — so real bracketed text ("[1]", "[see the docs]") survives.
+_AUDIO_TAG_RE = re.compile(
+    r"\[\s*(?:"
+    r"[a-z]+(?:ly|ing)"
+    r"|laughs?|chuckles?|sighs?|exhales?|inhales?|gulps?|pause|beat"
+    r"|clears throat|shouts?|whispers?|cries|sings?|gasps?|snorts?"
+    r"|sarcastic|excited|curious|nervous|angry|sad|happy|deadpan|flat"
+    r")\s*\]",
+    re.IGNORECASE,
+)
+
+
+def strip_audio_tags(text: str) -> str:
+    """Remove v3 delivery tags from text destined for the ordinary voice."""
+    return _AUDIO_TAG_RE.sub(" ", str(text or ""))
+
+
 def for_speech(text: str) -> str:
     """Plain prose for the TTS engine: no markdown, no emoji, no bare URLs,
     no stray symbols that would be read out character by character."""
-    text = (text or "").strip()
+    text = strip_audio_tags((text or "").strip())
     if not text:
         return ""
     for symbol, spoken in _PRE_SPOKEN_SYMBOLS.items():
@@ -136,8 +160,11 @@ def is_question(text: str) -> bool:
 
 def for_display(text: str) -> str:
     """What the speech bubble shows: markdown syntax removed (the bubble
-    can't render it) but emoji and layout-ish punctuation left alone."""
+    can't render it) but emoji and layout-ish punctuation left alone.
-    text = (text or "").strip()
+
+    Delivery tags go too — the voice no longer says them, so showing them
+    would caption a laugh nobody heard."""
+    text = strip_audio_tags((text or "").strip())
     if not text:
         return ""
     text = _strip_markdown(text, keep_emoji=True)

bolt_pet/ui/app.py

@@ -11,13 +11,14 @@ from __future__ import annotations
 import sys
 
 from PySide6.QtCore import QThread
-from PySide6.QtWidgets import QApplication
+from PySide6.QtWidgets import QApplication, QMessageBox
 
 from .. import config, updater
 from ..controller import PetController
 from ..hotkey import GlobalHotkey
 from ..state import PetState
 from .history_window import HistoryWindow
+from .mic_picker import MicPickerDialog
 from .pet_window import PetWindow
 from .tray import PetTray
 from .wake_tuner import WakeTunerWindow
@@ -41,6 +42,8 @@ def run() -> int:
     thread.started.connect(controller.run)
     controller.state_changed.connect(lambda value: window.set_state(PetState(value)))
     controller.said.connect(window.say)
+    # Lip-sync: the loudness of the audio actually going to the speakers.
+    controller.mouth.connect(window.set_mouth)
     controller.log.connect(_log)
     controller.action.connect(window.apply_action)  # petctl move/emote/say/...
     controller.finished.connect(thread.quit)
@@ -55,6 +58,19 @@ def run() -> int:
     window.publish_monitors()
     window.copied.connect(lambda text: _log(f"Copied to clipboard: {text[:60]}"))
 
+    # The updater has already moved the checkout by the time this fires; all
+    # that's left is to let Qt tear down cleanly (so the mic and the tray
+    # icon are released) and then exec the new code. Doing the exec after
+    # app.exec() returns, rather than from the controller thread, is what
+    # guarantees the audio device is free before the new process opens it.
+    # Defined early so the mic picker (below) can trigger the same restart
+    # a self-update or self-restart does.
+    pending_restart = {"tag": None}
+
+    def _handle_restart(tag: str) -> None:
+        pending_restart["tag"] = tag
+        app.quit()
+
     history_window = HistoryWindow(controller.history)
     tuner_window = WakeTunerWindow(
         get_threshold=controller.wake_threshold,
@@ -63,6 +79,23 @@ def run() -> int:
         on_reset=controller.reset_wake_stats,
     )
 
+    def _save_mic_device(value) -> None:
+        config.set_env_value("MIC_DEVICE", str(value) if value is not None else "")
+        _log(f"Microphone set to device #{value}." if value is not None else
+             "Microphone set to system default.")
+        answer = QMessageBox.question(
+            None, "Restart Bolt?",
+            "Restart now so Bolt starts listening on the new microphone?",
+            QMessageBox.Yes | QMessageBox.No,
+        )
+        if answer == QMessageBox.Yes:
+            _handle_restart("microphone change")
+
+    mic_picker = MicPickerDialog(
+        get_current=lambda: config.MIC_DEVICE,
+        on_save=_save_mic_device,
+    )
+
     def _set_nap(napping: bool) -> None:
         # Clicking the tray item pins the state; the schedule takes over again
         # only after a restart or a `petctl nap off`.
@@ -80,6 +113,7 @@ def run() -> int:
         on_set_nap=_set_nap,
         on_show_history=history_window.show_refreshed,
         on_show_wake_tuner=tuner_window.show_refreshed,
+        on_show_mic_picker=mic_picker.show_refreshed,
         on_reset_voice=controller.reset_voice,
     )
 
@@ -102,17 +136,6 @@ def run() -> int:
     elif hotkey.running:
         _log(f"Push-to-talk: {config.PUSH_TO_TALK_HOTKEY}")
 
-    # The updater has already moved the checkout by the time this fires; all
-    # that's left is to let Qt tear down cleanly (so the mic and the tray
-    # icon are released) and then exec the new code. Doing the exec after
-    # app.exec() returns, rather than from the controller thread, is what
-    # guarantees the audio device is free before the new process opens it.
-    pending_restart = {"tag": None}
-
-    def _handle_restart(tag: str) -> None:
-        pending_restart["tag"] = tag
-        app.quit()
-
     controller.restart_requested.connect(_handle_restart)
 
     def _shutdown() -> None:

bolt_pet/ui/mic_picker.py

@@ -0,0 +1,98 @@
+"""Microphone picker — MIC_DEVICE in .env is a device index most people have
+no way to look up short of running a script. This dialog lists whatever
+sounddevice can actually see, lets you click one, and writes its index back
+to .env so it survives a restart.
+
+The index, not the name, is what gets saved: the same device is commonly
+listed once per host API (MME, DirectSound, WASAPI, WDM-KS, ...) under the
+same or a near-identical name, which makes a bare name an ambiguous pick that
+sounddevice will flatly refuse to resolve (see audio/mic.list_input_devices).
+The index is exact.
+
+The mic is only ever opened once, at startup (see controller.run()), so a
+pick made here still needs a restart before Bolt actually uses it — this
+dialog just makes finding and saving the right device painless, and offers
+to do the restart for you.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Optional
+
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import (
+    QDialog, QDialogButtonBox, QHBoxLayout, QLabel, QListWidget,
+    QListWidgetItem, QPushButton, QVBoxLayout,
+)
+
+from ..audio.mic import list_input_devices
+
+_DEFAULT_LABEL = "System default"
+
+
+class MicPickerDialog(QDialog):
+    def __init__(
+        self,
+        get_current: Callable[[], Optional[int]],
+        on_save: Callable[[Optional[int]], None],
+    ):
+        super().__init__()
+        self._get_current = get_current
+        self._on_save = on_save
+
+        self.setWindowTitle("Bolt — select microphone")
+        self.resize(440, 320)
+
+        self._list = QListWidget()
+        refresh_button = QPushButton("Refresh")
+        refresh_button.clicked.connect(self.refresh)
+
+        top = QHBoxLayout()
+        top.addWidget(QLabel("Pick the microphone Bolt should listen on:"))
+        top.addStretch(1)
+        top.addWidget(refresh_button)
+
+        buttons = QDialogButtonBox(QDialogButtonBox.Save | QDialogButtonBox.Cancel)
+        buttons.accepted.connect(self._save)
+        buttons.rejected.connect(self.reject)
+
+        layout = QVBoxLayout(self)
+        layout.addLayout(top)
+        layout.addWidget(self._list)
+        layout.addWidget(buttons)
+
+    def refresh(self) -> None:
+        self._list.clear()
+        default_item = QListWidgetItem(_DEFAULT_LABEL)
+        default_item.setData(Qt.UserRole, None)
+        self._list.addItem(default_item)
+
+        current = self._get_current()
+        selected = default_item
+        try:
+            devices = list_input_devices()
+        except Exception as exc:
+            self._list.addItem(f"Could not list devices: {exc}")
+            devices = []
+
+        for device in devices:
+            item = QListWidgetItem(
+                f"{device['name']}  [{device['hostapi']}]  (#{device['index']})"
+            )
+            item.setData(Qt.UserRole, device["index"])
+            self._list.addItem(item)
+            if current == device["index"]:
+                selected = item
+        self._list.setCurrentItem(selected)
+
+    def _save(self) -> None:
+        item = self._list.currentItem()
+        value = item.data(Qt.UserRole) if item is not None else None
+        self._on_save(value)
+        self.accept()
+
+    def show_refreshed(self) -> None:
+        self.refresh()
+        self.show()
+        self.raise_()
+        self.activateWindow()

bolt_pet/ui/pet_window.py

@@ -19,7 +19,7 @@ from PySide6.QtGui import (
 )
 from PySide6.QtWidgets import QApplication, QWidget
 
-from .. import config
+from .. import config, window_state
 from ..monitors import Monitor
 from ..state import PetState
 from .sprite import WALK, SpriteSet
@@ -36,6 +36,11 @@ _NAP_OPACITY = 0.35
 # and the feet skate whenever PET_WANDER_SPEED doesn't happen to match the fps.
 # Eight frames at 13px is a ~104px stride cycle, a bit under the pet's width.
 _WALK_PIXELS_PER_FRAME = 13.0
+# How long a loudness level stays believable. The audio thread sends one per
+# ~30ms while a clip plays; if they stop arriving (offline TTS has no envelope,
+# or playback died) the mouth must not stay frozen mid-syllable, so after this
+# long the ordinary looping animation takes back over.
+_MOUTH_STALE_SECONDS = 0.35
 
 
 def emote_transform(emote: str, progress: float) -> tuple[float, float, float, float]:
@@ -213,6 +218,9 @@ class PetWindow(QWidget):
         self._commanded_move = False  # a petctl move — happens even mid-conversation
         self._next_wander_at = 0.0
         self._bob_offset = 0
+        # Lip-sync: loudness of what is playing right now, and when it arrived.
+        self._mouth_level: Optional[float] = None
+        self._mouth_at = 0.0
         self._bob_phase = 0.0
         self._walking = False
         self._facing = 1  # +1 right, -1 left; the walk art is drawn facing right
@@ -249,6 +257,19 @@ class PetWindow(QWidget):
             y = int(config.PET_START_Y) if config.PET_START_Y else None
         except ValueError:
             x = y = None
+        if x is None and y is None and config.PET_REMEMBER_POSITION:
+            remembered = window_state.load()
+            # Only if it still lands on a screen that exists — the usual reason
+            # a saved position is stale is that the monitor it was on has been
+            # unplugged, and restoring it faithfully would hide the pet.
+            if remembered is not None:
+                rectangles = [
+                    (s.availableGeometry().left(), s.availableGeometry().top(),
+                     s.availableGeometry().right(), s.availableGeometry().bottom())
+                    for s in QApplication.screens()
+                ]
+                if window_state.is_visible_on(*remembered, self.width(), rectangles):
+                    x, y = remembered
         if geo is not None:
             x = geo.right() - self.width() - 40 if x is None else x
             y = geo.bottom() - self.height() - 60 if y is None else y
@@ -688,6 +709,34 @@ class PetWindow(QWidget):
 
     # ── animation ────────────────────────────────────────────────────────
 
+    def set_mouth(self, level: float) -> None:
+        """How loud the pet is *right now* (0..1), straight off the PCM going
+        to the speakers (audio/tts.level_of).
+
+        The talking frames are ordered by mouth openness, so this indexes them
+        directly: the mouth moves with the actual waveform instead of flapping
+        on a timer, which is the difference between a talking sprite and a
+        dubbed one."""
+        self._mouth_level = max(0.0, min(1.0, float(level)))
+        self._mouth_at = time.monotonic()
+        if self._current_state == PetState.TALKING:
+            self.update()
+
+    def _mouth_frame(self, animation) -> Optional[QPixmap]:
+        """The frame matching the current loudness, or None to use the timer.
+
+        Falls back the moment the levels go stale — offline TTS has no
+        envelope, and a mouth frozen mid-syllable is worse than a timed loop."""
+        if self._mouth_level is None or animation is None:
+            return None
+        if time.monotonic() - self._mouth_at > _MOUTH_STALE_SECONDS:
+            return None
+        frames = animation.frames
+        if len(frames) < 2:
+            return None
+        index = int(round(self._mouth_level * (len(frames) - 1)))
+        return frames[max(0, min(len(frames) - 1, index))]
+
     def _advance_frame(self) -> None:
         # While walking the cycle is stepped by _advance_walk from distance
         # travelled; letting this timer also advance it would double-step it
@@ -702,7 +751,12 @@ class PetWindow(QWidget):
         painter.setRenderHint(QPainter.Antialiasing)
         painter.setRenderHint(QPainter.SmoothPixmapTransform)
         key = self._animation_key()
-        pixmap: Optional[QPixmap] = self.sprites.get(key).current()
+        animation = self.sprites.get(key)
+        pixmap: Optional[QPixmap] = None
+        if key == PetState.TALKING:
+            pixmap = self._mouth_frame(animation)
+        if pixmap is None:
+            pixmap = animation.current()
         if key == WALK:
             pixmap = self._oriented(pixmap)
         if pixmap is None:
@@ -759,6 +813,9 @@ class PetWindow(QWidget):
         was_click = not self._dragged
         self._drag_offset = None
         self._press_pos = None
+        if not was_click and config.PET_REMEMBER_POSITION:
+            position = self.geometry().topLeft()
+            window_state.save(position.x(), position.y())
         if was_click:
             self.talk_requested.emit()
         else:

bolt_pet/ui/tray.py

@@ -47,6 +47,7 @@ class PetTray(QSystemTrayIcon):
         on_set_nap: Optional[Callable[[bool], None]] = None,
         on_show_history: Optional[Callable[[], None]] = None,
         on_show_wake_tuner: Optional[Callable[[], None]] = None,
+        on_show_mic_picker: Optional[Callable[[], None]] = None,
         on_reset_voice: Optional[Callable[[], None]] = None,
         parent=None,
     ):
@@ -104,6 +105,11 @@ class PetTray(QSystemTrayIcon):
             tuner_action.triggered.connect(on_show_wake_tuner)
             menu.addAction(tuner_action)
 
+        if on_show_mic_picker is not None:
+            mic_action = QAction("Select microphone…", menu)
+            mic_action.triggered.connect(on_show_mic_picker)
+            menu.addAction(mic_action)
+
         # Only ever enabled while a server-picked voice (speak_as) is in use —
         # it's the way back from "talk like a pirate", which nothing else
         # undoes short of a restart.

bolt_pet/window_state.py

@@ -0,0 +1,75 @@
+"""Where the pet was left, so it starts there next time.
+
+Listed in the README as a known limitation: drag it somewhere deliberate, and
+the next launch puts it back in the bottom-right corner. For something that
+lives on your desktop all day that is a small daily annoyance, and it is a
+config write on drag-end.
+
+Lives in the cache dir rather than the repo, next to the restart context: it
+is per-machine state about this install, not something that belongs in a git
+diff. Every function swallows its own errors — a corrupt or unwritable state
+file must never stop the pet from starting, it just means the default corner.
+
+Positions are validated against the *current* screen layout on load, because
+the common case for a stale position is exactly the case where it is
+dangerous: the pet was last on a monitor that is now unplugged, and restoring
+it faithfully would put it somewhere you cannot see or reach.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("bolt_pet.window_state")
+
+DEFAULT_PATH = Path.home() / ".cache" / "bolt-pet" / "window.json"
+
+
+def load(path: Optional[Path] = None) -> Optional[tuple[int, int]]:
+    """The saved position, or None if there isn't a usable one."""
+    try:
+        data = json.loads(Path(path or DEFAULT_PATH).read_text(encoding="utf-8"))
+        return int(data["x"]), int(data["y"])
+    except (FileNotFoundError, json.JSONDecodeError, KeyError, TypeError, ValueError, OSError):
+        return None
+
+
+def save(x: int, y: int, path: Optional[Path] = None) -> None:
+    """Remember where it is now. Atomic, so a crash mid-write can't leave a
+    half-file that makes the next start fall back to the corner."""
+    target = Path(path or DEFAULT_PATH)
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        descriptor, temp_path = tempfile.mkstemp(dir=target.parent, prefix=".window_", suffix=".tmp")
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                json.dump({"x": int(x), "y": int(y)}, handle)
+            os.replace(temp_path, target)
+        except BaseException:
+            try:
+                os.unlink(temp_path)
+            except OSError:
+                pass
+            raise
+    except Exception:
+        logger.debug("Could not save the window position", exc_info=True)
+
+
+def is_visible_on(x: int, y: int, size: int, rectangles) -> bool:
+    """Whether that position still lands on a screen that exists.
+
+    *rectangles* are (left, top, right, bottom) tuples — the caller's job,
+    because this module has no business importing Qt. Requires a real overlap
+    rather than a touching edge, so a pet saved flush against the boundary of a
+    monitor that has since been unplugged is not counted as reachable."""
+    for left, top, right, bottom in rectangles:
+        overlap_x = min(x + size, right) - max(x, left)
+        overlap_y = min(y + size, bottom) - max(y, top)
+        if overlap_x > size * 0.25 and overlap_y > size * 0.25:
+            return True
+    return False

requirements.txt

@@ -8,6 +8,11 @@ numpy>=1.24
 
 # HTTP client to the Bolt desk API
 requests>=2.31
+# Speech-to-text, via the server's /desk/stt websocket relay (audio/stt.py,
+# audio/stt_stream.py) — there is no separate REST fallback, so this is now
+# load-bearing for transcription to work at all, not just for the
+# transcribe-while-talking optimisation STT_STREAMING controls.
+websocket-client>=1.7
 
 # Wake-word detection (local, offline after first run) — runs the
 # custom-trained thunderbolt.onnx model shipped in this repo, same runtime
@@ -16,7 +21,7 @@ requests>=2.31
 # under the package's own resources/ dir afterward) — needs internet once.
 openwakeword
 
-# Offline TTS fallback if ElevenLabs isn't configured or a request fails.
+# Offline TTS fallback if the server call isn't configured or fails.
 # Uses SAPI5 on Windows, NSSpeechSynthesizer on macOS, espeak on Linux
 # (Linux also needs: sudo apt install espeak-ng).
 pyttsx3>=2.90

scripts/generate_bolt_sprites.py

@@ -663,31 +663,43 @@ def render_frame(p) -> Image.Image:
 
 def frames_for(state: str) -> list[dict]:
     if state == "idle":
+        # Eight frames, all distinct. The old version drove breathing on
+        # sin(2*pi*t) and the tail on sin(4*pi*t), which both cross zero at
+        # i=0 and i=4 — so frame 4 was byte-identical to frame 0 and the loop
+        # was really four frames stored twice.
         out = []
         for i in range(8):
             t = i / 8
-            br = math.sin(t * 2 * math.pi)
+            br = math.sin(t * 2 * math.pi + math.pi / 7)
             out.append(
                 default_pose(
                     breathe=br,
                     head_dy=-0.006 * br,
-                    tail=math.sin(t * 4 * math.pi),
+                    # Three-halves harmonic: never in phase with the breath, so
+                    # no two frames of the cycle can coincide.
+                    tail=math.sin(t * 3 * math.pi + 0.6),
+                    ear_twitch=0.30 if i == 3 else 0.0,   # a flick, once a loop
                     blink=1.0 if i == 6 else 0.0,
                 )
             )
         return out
     if state == "listening":
+        # Six frames of *orienting*, not idling: ears up, head turning toward
+        # whoever is talking, then settling. The old four had frames 0 and 2
+        # differing by 0.09 mean pixels — a two-pose animation wearing four.
         out = []
-        for i in range(4):
+        for i in range(6):
-            t = i / 4
+            t = i / 6
+            lean = math.sin(t * 2 * math.pi + math.pi / 5)
             out.append(
                 default_pose(
                     ear=1.0,
-                    ear_twitch=0.35 * math.sin(t * 2 * math.pi),
+                    ear_twitch=0.45 * math.sin(t * 4 * math.pi),
-                    tilt=-7 + 2.0 * math.sin(t * 2 * math.pi),
+                    tilt=-9 + 4.0 * lean,
+                    look=(0.010 * lean, -0.004),
                     brow=1.0,
-                    tail=0.5 * math.sin(t * 2 * math.pi),
+                    tail=0.7 * math.sin(t * 2 * math.pi + 1.1),
-                    head_dy=-0.008,
+                    head_dy=-0.010 - 0.004 * lean,
                     tag_glow=True,
                     extras="listen",
                     phase=i,
@@ -695,35 +707,51 @@ def frames_for(state: str) -> list[dict]:
             )
         return out
     if state == "thinking":
+        # The old six moved by a mean of ~1.3 pixels — effectively a still
+        # image. Thinking should *look* like thinking: the head tilts, the eyes
+        # travel as if following a thought, and one ear rotates independently.
         out = []
         for i in range(6):
             t = i / 6
+            sway = math.sin(t * 2 * math.pi)
             out.append(
                 default_pose(
-                    ear=0.25,
+                    ear=0.25 + 0.35 * abs(sway),
-                    tilt=6.0,
+                    ear_twitch=0.5 * math.cos(t * 2 * math.pi),
-                    look=(0.022, -0.026),
+                    tilt=4.0 + 7.0 * sway,
-                    brow=0.5,
+                    # Eyes wander a small circle: the cheapest possible read of
-                    breathe=0.4 * math.sin(t * 2 * math.pi),
+                    # "working something out" and the thing most obviously
-                    tail=0.2 * math.sin(t * 2 * math.pi),
+                    # missing before.
+                    look=(0.026 * math.cos(t * 2 * math.pi),
+                          -0.020 + 0.014 * math.sin(t * 2 * math.pi)),
+                    brow=0.5 + 0.4 * abs(sway),
+                    breathe=0.5 * math.sin(t * 2 * math.pi + 0.9),
+                    head_dy=-0.008 * sway,
+                    tail=0.35 * math.sin(t * 3 * math.pi),
+                    blink=1.0 if i == 4 else 0.0,
                     extras="think",
                     phase=i // 2,
                 )
             )
         return out
     if state == "talking":
+        # Ordered by mouth openness — closed at frame 0, widest at the last —
+        # because the window indexes these by the loudness of the audio that is
+        # actually playing (see audio/tts.level_of and PetWindow.set_mouth).
+        # A time-ordered loop cannot be indexed that way, and a mouth that
+        # flaps on a timer is what makes a talking sprite look dubbed.
         out = []
-        for i in range(4):
+        count = 6
-            t = i / 4
+        for i in range(count):
-            open_ = (math.sin(t * 2 * math.pi) + 1) / 2
+            open_ = i / (count - 1)
             out.append(
                 default_pose(
-                    mouth=0.25 + 0.75 * open_,
+                    mouth=0.06 + 0.94 * open_,
                     ear=0.6,
                     head_dy=-0.010 * open_,
                     breathe=open_,
-                    tail=math.sin(t * 2 * math.pi + 1.0),
+                    tail=0.45 * math.sin(i * 0.9),
-                    brow=0.35,
+                    brow=0.35 * open_,
                 )
             )
         return out

tests/test_config.py

@@ -0,0 +1,62 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet import config
+
+
+def test_set_env_value_uncomments_existing_line(tmp_path, monkeypatch):
+    (tmp_path / ".env").write_text("#MIC_DEVICE=\nOTHER=1\n", encoding="utf-8")
+    monkeypatch.setattr(config, "HERE", tmp_path)
+
+    config.set_env_value("MIC_DEVICE", "USB Mic")
+
+    lines = (tmp_path / ".env").read_text(encoding="utf-8").splitlines()
+    assert lines == ["MIC_DEVICE=USB Mic", "OTHER=1"]
+    assert config.os.environ["MIC_DEVICE"] == "USB Mic"
+
+
+def test_set_env_value_overwrites_existing_uncommented_line(tmp_path, monkeypatch):
+    (tmp_path / ".env").write_text("MIC_DEVICE=Old Mic\n", encoding="utf-8")
+    monkeypatch.setattr(config, "HERE", tmp_path)
+
+    config.set_env_value("MIC_DEVICE", "New Mic")
+
+    lines = (tmp_path / ".env").read_text(encoding="utf-8").splitlines()
+    assert lines == ["MIC_DEVICE=New Mic"]
+
+
+def test_set_env_value_appends_when_key_is_missing(tmp_path, monkeypatch):
+    (tmp_path / ".env").write_text("OTHER=1\n", encoding="utf-8")
+    monkeypatch.setattr(config, "HERE", tmp_path)
+
+    config.set_env_value("MIC_DEVICE", "New Mic")
+
+    lines = (tmp_path / ".env").read_text(encoding="utf-8").splitlines()
+    assert lines == ["OTHER=1", "MIC_DEVICE=New Mic"]
+
+
+def test_set_env_value_creates_file_if_missing(tmp_path, monkeypatch):
+    monkeypatch.setattr(config, "HERE", tmp_path)
+
+    config.set_env_value("MIC_DEVICE", "New Mic")
+
+    assert (tmp_path / ".env").read_text(encoding="utf-8") == "MIC_DEVICE=New Mic\n"
+
+
+def test_mic_device_parses_a_numeric_index_as_int():
+    # sounddevice only takes an int index as-is; a string (even "1") goes
+    # through substring matching against every device name instead, which is
+    # not what a saved index means — see the tray's mic picker.
+    assert config._mic_device("1") == 1
+    assert isinstance(config._mic_device("1"), int)
+
+
+def test_mic_device_keeps_a_name_as_a_string():
+    assert config._mic_device("USB Mic") == "USB Mic"
+
+
+def test_mic_device_blank_is_none():
+    assert config._mic_device("") is None
+    assert config._mic_device("   ") is None

tests/test_controller.py

@@ -22,6 +22,14 @@ def no_screen_probes(monkeypatch):
     monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False)
 
 
+@pytest.fixture(autouse=True)
+def _no_streaming(monkeypatch):
+    """Most tests drive the plain request/response path; leaving streaming on
+    would have them attempt a real HTTP call, fail, and fall back — passing,
+    slowly, for the wrong reason. The streaming path has its own tests."""
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
+
+
 @pytest.fixture
 def ctrl():
     return controller_mod.PetController()
@@ -41,10 +49,10 @@ def test_full_turn_happy_path(monkeypatch, ctrl):
 
     monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16))
     monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's the weather")
-    monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None: controller_mod.server_client.Reply("sunny and 72"))
+    monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None, on_say=None: controller_mod.server_client.Reply("sunny and 72"))
     spoken = []
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1])
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1])
 
     ctrl._handle_conversation_turn()
 
@@ -59,7 +67,7 @@ def test_turn_with_nothing_heard_returns_to_idle_without_calling_server(monkeypa
     monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: None)
     called = {"n": 0}
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: called.__setitem__("n", called["n"] + 1))
+                        lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1))
 
     ctrl._handle_conversation_turn()
 
@@ -97,7 +105,7 @@ def test_turn_with_server_error_flashes_error_then_idle(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16))
     monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "hello")
 
-    def boom(text, on_command=None):
+    def boom(text, on_command=None, on_say=None):
         raise controller_mod.server_client.ServerError("server is down")
     monkeypatch.setattr(controller_mod.server_client, "converse", boom)
 
@@ -150,7 +158,7 @@ def test_heartbeat_speaks_a_pending_announcement_when_idle(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: "don't forget your 3pm")
     spoken = []
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1])
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1])
     said = _capture(ctrl.said)
 
     ctrl._maybe_heartbeat()

tests/test_controller_features.py

@@ -29,6 +29,14 @@ def no_screen_probes(monkeypatch):
     monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text)
 
 
+@pytest.fixture(autouse=True)
+def _no_streaming(monkeypatch):
+    """Most tests drive the plain request/response path; leaving streaming on
+    would have them attempt a real HTTP call, fail, and fall back — passing,
+    slowly, for the wrong reason. The streaming path has its own tests."""
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
+
+
 @pytest.fixture
 def ctrl():
     return controller_mod.PetController()
@@ -160,7 +168,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch
     ctrl._barge_in = detector
     logs = _capture(ctrl.log)
 
-    def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None):
+    def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None, on_level=None):
         # What really happens: frames get scored during playback, then one
         # clears the threshold and playback aborts.
         detector._frames, detector._peak, detector._last = 7, 0.81, 0.81
@@ -180,7 +188,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch
 def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl):
     logs = _capture(ctrl.log)
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: False)  # interrupted
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False)  # interrupted
 
     ctrl._speak("a very long explanation")
 
@@ -190,7 +198,7 @@ def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl):
 
 def test_uninterrupted_playback_does_not_queue_a_turn(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
     ctrl._speak("short answer")
     assert not ctrl._talk_now.is_set()
 
@@ -202,7 +210,7 @@ def spoke(monkeypatch):
     """Playback that always completes, so only the follow-up rule decides
     whether another turn is queued."""
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
 
 
 def test_a_reply_ending_in_a_question_keeps_listening(spoke, ctrl):
@@ -290,7 +298,7 @@ def test_follow_up_can_be_turned_off(spoke, monkeypatch, ctrl):
 
 def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: False)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False)
     ctrl._follow_ups = 3
 
     ctrl._speak("a very long explanation")
@@ -301,7 +309,7 @@ def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl):
 
 def test_speech_is_recorded_in_the_history(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
     ctrl._speak("**bold** reply")
     assert ctrl.history.last().text == "**bold** reply"  # raw, for copy/paste
 
@@ -315,10 +323,10 @@ def test_the_active_window_rides_along_with_the_utterance(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.screen_context, "context_for",
                         lambda text: f"{text}\n\n[on screen right now: app.py]")
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
     sent = []
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: sent.append(text) or Reply("that's a KeyError"))
+                        lambda text, on_command=None, on_say=None: sent.append(text) or Reply("that's a KeyError"))
 
     ctrl._handle_conversation_turn()
 
@@ -371,10 +379,10 @@ def test_napping_still_answers_when_spoken_to(monkeypatch, ctrl):
                         lambda *a, **k: np.zeros(10, dtype=np.int16))
     monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "you awake?")
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: Reply("always"))
+                        lambda text, on_command=None, on_say=None: Reply("always"))
     spoken = []
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True)
     ctrl.set_napping(True)
 
     ctrl._handle_conversation_turn()
@@ -389,9 +397,9 @@ def test_notifications_are_forwarded_and_spoken(monkeypatch, ctrl):
     ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
     sent, spoken = [], []
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: sent.append(text) or Reply("your build is green"))
+                        lambda text, on_command=None, on_say=None: sent.append(text) or Reply("your build is green"))
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True)
 
     ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
     ctrl._drain_notifications()
@@ -411,7 +419,7 @@ def test_notifications_are_not_forwarded_while_napping(monkeypatch, ctrl):
     ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
     called = {"n": 0}
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: called.__setitem__("n", called["n"] + 1))
+                        lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1))
     ctrl.set_napping(True)
 
     ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
@@ -507,9 +515,9 @@ def test_check_deliveries_runs_after_a_conversation_turn(monkeypatch, ctrl, tmp_
                         lambda *a, **k: np.zeros(10, dtype=np.int16))
     monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "send me that file")
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: Reply("it's on the way"))
+                        lambda text, on_command=None, on_say=None: Reply("it's on the way"))
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None: True)
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
     monkeypatch.setattr(controller_mod.config, "DELIVERED_FILES_DIR", tmp_path)
     monkeypatch.setattr(controller_mod.server_client, "list_outbox_files",
                         lambda: [{"id": "abc", "name": "notes.txt", "size": 2}])
@@ -531,10 +539,10 @@ def _voice_turn(monkeypatch, ctrl, reply, said="talk like a pirate"):
                         lambda *a, **k: np.zeros(10, dtype=np.int16))
     monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: said)
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: reply)
+                        lambda text, on_command=None, on_say=None: reply)
     monkeypatch.setattr(
         controller_mod.tts, "speak",
-        lambda text, on_error=None, should_stop=None, voice_id=None:
+        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
             voices.append(voice_id) or True,
     )
     ctrl._handle_conversation_turn()
@@ -625,7 +633,7 @@ def test_dialoguectl_never_reaches_the_shell(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
                         lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
     monkeypatch.setattr(controller_mod.tts, "play_pcm",
-                        lambda pcm, rate, should_stop=None: True)
+                        lambda pcm, rate, should_stop=None, on_level=None: True)
 
     output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "[cheerfully] hi"}))
 
@@ -638,7 +646,7 @@ def test_a_scene_shows_in_the_bubble_with_the_delivery_tags_stripped(monkeypatch
     monkeypatch.setattr(controller_mod.config, "DIALOGUE_VOICES", "narrator:9BWtsMINqrJLrRacOk9x")
     monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
                         lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
-    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
+    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
     said = _capture(ctrl.said)
 
     ctrl._handle_command(_dialogue_command(
@@ -656,7 +664,7 @@ def test_a_mid_turn_scene_returns_to_thinking_not_idle(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.config, "ELEVENLABS_VOICE_ID", "aaorr6ZHIL88gEexu7dC")
     monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
                         lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
-    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
+    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
     ctrl._state.transition(PetState.LISTENING)
     ctrl._state.transition(PetState.THINKING)
     states = _capture(ctrl.state_changed)
@@ -676,7 +684,7 @@ def test_the_scene_uses_a_voice_the_server_picked_with_speak_as(monkeypatch, ctr
         return np.zeros(4, dtype=np.int16), 24000
 
     monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", capture)
-    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
+    monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
     ctrl._apply_voice(controller_mod.server_client.Reply("ok", "PICKEDvoice123456789", "Terence"))
 
     ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"}))
@@ -718,7 +726,7 @@ def test_talking_over_a_scene_is_reported_up_the_relay(monkeypatch, ctrl):
     monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
                         lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
     monkeypatch.setattr(controller_mod.tts, "play_pcm",
-                        lambda pcm, rate, should_stop=None: False)  # barge-in
+                        lambda pcm, rate, should_stop=None, on_level=None: False)  # barge-in
 
     output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"}))
 
@@ -788,9 +796,9 @@ def test_coming_back_up_reports_to_the_server_and_speaks_the_reply(monkeypatch,
                                     path=state, now=1000.0)
     sent, spoken = [], []
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: sent.append(text) or Reply("Good, it's up."))
+                        lambda text, on_command=None, on_say=None: sent.append(text) or Reply("Good, it's up."))
     monkeypatch.setattr(controller_mod.tts, "speak",
-                        lambda text, on_error=None, should_stop=None, voice_id=None:
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
                             spoken.append(text) or True)
 
     ctrl._report_self_restart()
@@ -805,8 +813,230 @@ def test_an_ordinary_start_reports_nothing(monkeypatch, ctrl, tmp_path):
     monkeypatch.setattr(controller_mod.self_restart, "DEFAULT_STATE_PATH", tmp_path / "none.json")
     called = []
     monkeypatch.setattr(controller_mod.server_client, "converse",
-                        lambda text, on_command=None: called.append(text))
+                        lambda text, on_command=None, on_say=None: called.append(text))
 
     ctrl._report_self_restart()
 
     assert called == []
+
+
+# ── holding line: "give me a sec" while a tool runs ────────────────────────
+# The server used to discard whatever the model wrote alongside a tool call,
+# so the whole round trip was silence — and the model, with no evidence its
+# sentence landed, said it again in the final reply.
+
+def test_a_holding_line_is_spoken_before_the_tool_runs(monkeypatch, ctrl):
+    order = []
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
+                            order.append(("spoke", text)) or True)
+    monkeypatch.setattr(controller_mod.server_client, "run_local_command",
+                        lambda cmd: order.append(("ran", cmd)) or "[exit 0]")
+
+    ctrl._speak_holding("Give me a sec.")
+    ctrl._handle_command("df -h /")
+
+    assert order == [("spoke", "Give me a sec."), ("ran", "df -h /")]
+
+
+def test_the_holding_line_shows_in_the_bubble_but_not_the_transcript(monkeypatch, ctrl):
+    """It's filler. The transcript should keep the actual answer."""
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
+    said = _capture(ctrl.said)
+
+    ctrl._speak_holding("I'll check on that — one moment.")
+
+    assert said == ["I'll check on that — one moment."]
+    assert ctrl.history.entries() == []
+
+
+def test_a_holding_line_returns_to_waiting_not_idle(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
+    ctrl._state.transition(PetState.LISTENING)
+    ctrl._state.transition(PetState.THINKING)
+    states = _capture(ctrl.state_changed)
+
+    ctrl._speak_holding("one sec")
+
+    assert states == ["talking", "thinking"]
+
+
+def test_the_relay_speaks_whatever_the_server_attaches(monkeypatch, ctrl):
+    spoken = []
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
+                            spoken.append(text) or True)
+    monkeypatch.setattr(controller_mod.mic, "record_utterance",
+                        lambda *a, **k: np.zeros(10, dtype=np.int16))
+    monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's in that folder?")
+
+    def fake_converse(text, on_command=None, on_say=None):
+        on_say("Let me check that for you.")      # what the server now sends
+        on_command("filectl {\"op\": \"list\", \"path\": \"/tmp\"}")
+        return Reply("It's got three files in it.")
+
+    monkeypatch.setattr(controller_mod.server_client, "converse", fake_converse)
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["Let me check that for you.", "It's got three files in it."]
+
+
+# ── streamed replies ───────────────────────────────────────────────────────
+# Without streaming the pet waits out the entire model call before saying a
+# word. With it, the wait is time-to-first-sentence.
+
+def test_each_sentence_is_spoken_as_it_arrives(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
+    spoken = []
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
+                            spoken.append(text) or True)
+
+    def fake_stream(text, on_say, on_command=None, timeout=180.0):
+        on_say("The disk is fine.")
+        on_say("About sixty percent used.")
+        return controller_mod.server_client.Reply(
+            "The disk is fine. About sixty percent used.", spoken=True)
+
+    monkeypatch.setattr(controller_mod.server_client, "converse_stream", fake_stream)
+    monkeypatch.setattr(controller_mod.mic, "record_utterance",
+                        lambda *a, **k: np.zeros(10, dtype=np.int16))
+    monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "how's the disk?")
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["The disk is fine.", "About sixty percent used."]
+    # ...and the empty final reply is not spoken as an extra blank utterance.
+    assert ctrl.history.entries()[-1].text == "About sixty percent used."
+
+
+def test_a_stream_that_fails_before_speaking_falls_back_silently(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
+    spoken = []
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
+                            spoken.append(text) or True)
+
+    def broken_stream(text, on_say, on_command=None, timeout=180.0):
+        raise controller_mod.server_client.ServerError("no streaming endpoint")
+
+    monkeypatch.setattr(controller_mod.server_client, "converse_stream", broken_stream)
+    monkeypatch.setattr(controller_mod.server_client, "converse",
+                        lambda text, on_command=None, on_say=None: Reply("Fell back fine."))
+
+    assert ctrl._ask_server("hello").text == "Fell back fine."
+    assert spoken == []   # nothing was said twice
+
+
+def test_streaming_can_be_switched_off(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
+    called = []
+    monkeypatch.setattr(controller_mod.server_client, "converse_stream",
+                        lambda *a, **k: called.append(1))
+    monkeypatch.setattr(controller_mod.server_client, "converse",
+                        lambda text, on_command=None, on_say=None: Reply("plain path"))
+
+    assert ctrl._ask_server("hi").text == "plain path"
+    assert called == []
+
+
+def test_talking_over_a_streamed_reply_still_interrupts(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False)
+
+    ctrl._speak_stream_chunk("A long explanation you cut short.")
+
+    assert ctrl._talk_now.is_set()
+
+
+# ── notification latency (reported 2026-08-02: 5-10 minutes) ───────────────
+# Draining was wired to the heartbeat's 60s interval, and a heartbeat that
+# landed mid-conversation stamped its clock before noticing — so it burned the
+# slot and waited another full interval. Several of those in a row is minutes.
+
+def test_a_notification_goes_out_on_the_next_tick_not_the_next_heartbeat(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.config, "NOTIFICATION_MIN_INTERVAL_SECONDS", 0)
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
+    sent = []
+    monkeypatch.setattr(controller_mod.server_client, "converse",
+                        lambda text, on_command=None, on_say=None:
+                            sent.append(text) or Reply("noted"))
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
+    monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None)
+    # A heartbeat has just run, so the next one is a full interval away.
+    ctrl._last_heartbeat = controller_mod.time.monotonic()
+
+    ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
+    ctrl._maybe_heartbeat()
+
+    assert sent and "Harry" in sent[0]
+
+
+def test_a_heartbeat_skipped_mid_conversation_does_not_burn_its_slot(monkeypatch, ctrl):
+    monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None)
+    ctrl._last_heartbeat = 0.0
+    ctrl._state.transition(PetState.LISTENING)
+    ctrl._state.transition(PetState.THINKING)
+
+    ctrl._maybe_heartbeat()          # due, but the pet is busy
+    assert ctrl._last_heartbeat == 0.0, "the clock must not advance on a skipped tick"
+
+    ctrl._state.transition(PetState.IDLE)
+    ctrl._maybe_heartbeat()          # free now — runs immediately
+    assert ctrl._last_heartbeat > 0.0
+
+
+def test_notifications_wait_while_the_pet_is_mid_turn(monkeypatch, ctrl):
+    """Speaking over the answer they're already getting would be worse than
+    waiting a couple of seconds."""
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
+    sent = []
+    monkeypatch.setattr(controller_mod.server_client, "converse",
+                        lambda text, on_command=None, on_say=None: sent.append(text))
+    ctrl._state.transition(PetState.LISTENING)
+
+    ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
+    ctrl._maybe_heartbeat()
+
+    assert sent == []
+    assert len(ctrl._pending_notifications) == 1   # kept, not dropped
+
+
+def test_a_second_message_inside_the_rate_limit_is_no_longer_lost(monkeypatch, ctrl):
+    """The gate used to drop it. With NOTIFICATION_MIN_INTERVAL_SECONDS=60,
+    two texts a minute apart meant you heard about one of them."""
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 60)
+    sent = []
+    monkeypatch.setattr(controller_mod.server_client, "converse",
+                        lambda text, on_command=None, on_say=None:
+                            sent.append(text) or Reply("ok"))
+    monkeypatch.setattr(controller_mod.tts, "speak",
+                        lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
+
+    ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
+    ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="it's urgent"))
+    ctrl._drain_notifications()
+
+    assert len(sent) == 1                      # one round trip...
+    assert "you there?" in sent[0] and "it's urgent" in sent[0]   # ...both messages
+    assert "2 desktop notifications" in sent[0]
+
+
+def test_the_filter_still_applies(ctrl):
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("deploy", 0)
+    ctrl._queue_notification(Notification(app="Chat", summary="lunch?", body=""))
+    assert ctrl._pending_notifications == []
+
+
+def test_a_notification_storm_cannot_become_an_unbounded_backlog(ctrl):
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
+    for index in range(40):
+        ctrl._queue_notification(Notification(app="Spam", summary=f"#{index}", body=""))
+
+    assert len(ctrl._pending_notifications) == controller_mod._MAX_PENDING_NOTIFICATIONS
+    assert "#39" in ctrl._pending_notifications[-1].as_text()   # newest kept

tests/test_dialogue.py

@@ -161,7 +161,18 @@ def test_the_relay_report_names_the_cast():
     action = dialogue.parse('dialoguectl ' + _scene(
         '{"voice": "self", "text": "one"}', '{"voice": "narrator", "text": "two"}',
     ))
-    assert dialogue.describe(action) == "[dialogue] played 2 lines in 2 voices: narrator, self"
+    assert dialogue.describe(action).startswith(
+        "[dialogue] played 2 lines in 2 voices: narrator, self")
+
+
+def test_the_report_says_the_scene_was_already_heard():
+    """Observed 2026-07-31: the scene played, then the final reply summarised
+    it, so the pet said the same thing twice with nothing in between. The
+    model cannot know the audio already happened unless it is told."""
+    action = dialogue.parse('dialoguectl {"lines": [{"text": "hello"}]}')
+    report = dialogue.describe(action)
+    assert "HEARD this already" in report
+    assert "Do not repeat" in report
 
 
 # ── the HTTP request ────────────────────────────────────────────────────────

tests/test_doctor.py

@@ -0,0 +1,187 @@
+"""The preflight. Its one job is to never be the thing that's broken.
+
+A doctor that raises on a broken install diagnoses the wrong patient, so the
+tests that matter here are the ugly-input ones: no config at all, a check that
+throws, a dependency missing. The individual diagnoses are simple enough to
+read; that they *run* on a machine missing everything is the property worth
+pinning down.
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet import config, doctor
+from bolt_pet.doctor import FAIL, OK, WARN
+
+
+def test_every_check_returns_a_verdict_on_a_bare_machine(monkeypatch):
+    """Nothing configured, nothing installed — still a full report."""
+    for name in ("SERVER_URL", "API_KEY", "ELEVENLABS_API_KEY", "ELEVENLABS_VOICE_ID"):
+        monkeypatch.setattr(config, name, "")
+    monkeypatch.setattr(doctor, "_module", lambda _n: False)
+
+    results = doctor.run()
+
+    assert len(results) == len(doctor.CHECKS)
+    assert all(c.status in (OK, WARN, FAIL) for c in results)
+    assert all(c.name and c.detail for c in results)
+
+
+def test_a_check_that_raises_does_not_hide_the_others():
+    """One broken probe must not cost you the other eleven diagnoses."""
+    def explode(*_args):
+        raise RuntimeError("boom")
+
+    original = doctor.CHECKS
+    doctor.CHECKS = (("mic", explode),) + original[:2]
+    try:
+        results = doctor.run()
+    finally:
+        doctor.CHECKS = original
+
+    assert len(results) == 3
+    assert results[0].status == FAIL
+    assert "boom" in results[0].detail
+
+
+def test_missing_server_config_is_a_failure_not_a_warning(monkeypatch):
+    """Without it the controller exits its thread at startup — the pet looks
+    alive and simply never answers. That is the worst failure mode there is."""
+    monkeypatch.setattr(config, "missing_config", lambda: ["BOLT_SERVER_URL", "DESK_API_KEY"])
+    check = doctor.check_config()
+    assert check.status == FAIL
+    assert "BOLT_SERVER_URL" in check.detail
+    assert check.fix
+
+
+def test_a_configured_server_passes_without_being_contacted(monkeypatch):
+    """The shallow run must not need the network — it's the first thing you
+    reach for when the network is what's wrong."""
+    monkeypatch.setattr(config, "missing_config", lambda: [])
+    monkeypatch.setattr(config, "SERVER_URL", "http://bolt.local:8000")
+    assert doctor.check_config().status == OK
+    assert doctor.check_server(deep=False).status == OK
+
+
+def test_every_problem_comes_with_something_to_do_about_it(monkeypatch):
+    """"screen reading: warn" is useless on its own; "apt install tesseract-ocr"
+    is the entire point of the tool."""
+    for name in ("SERVER_URL", "API_KEY"):
+        monkeypatch.setattr(config, name, "")
+    monkeypatch.setattr(doctor, "_module", lambda _n: False)
+
+    for check in doctor.run():
+        if check.status == FAIL:
+            assert check.fix, f"{check.name} says what's wrong but not what to do"
+
+
+def test_the_exit_code_is_nonzero_only_for_real_failures(monkeypatch, capsys):
+    monkeypatch.setattr(doctor, "run", lambda deep=False: [
+        doctor.Check("a", OK, "fine"), doctor.Check("b", WARN, "degraded")])
+    assert doctor.main([]) == 0
+
+    monkeypatch.setattr(doctor, "run", lambda deep=False: [doctor.Check("a", FAIL, "broken")])
+    assert doctor.main([]) == 1
+    assert "Fix those first" in capsys.readouterr().out
+
+
+def test_deep_is_off_unless_asked(monkeypatch):
+    seen = []
+    monkeypatch.setattr(doctor, "run", lambda deep=False: seen.append(deep) or [])
+    doctor.main([])
+    doctor.main(["--deep"])
+    assert seen == [False, True]
+
+
+def test_a_slow_silence_timeout_is_flagged(monkeypatch):
+    """The setting most likely to make it feel sluggish, and the least obvious
+    — it is pure dead air before anything at all starts happening."""
+    monkeypatch.setattr(config, "SILENCE_END_SEC", 2.0)
+    check = doctor.check_latency()
+    assert check.status == WARN
+    assert "2s" in check.detail or "2 " in check.detail
+
+    monkeypatch.setattr(config, "SILENCE_END_SEC", 0.9)
+    assert doctor.check_latency().status == OK
+
+
+def test_a_check_line_renders_the_fix_only_when_there_is_a_problem():
+    assert "→" not in doctor.Check("x", OK, "all good", fix="unused").line()
+    assert "→ do the thing" in doctor.Check("x", WARN, "hmm", fix="do the thing").line()
+
+
+# ── check_tts's --deep probe ────────────────────────────────────────────────
+# check_streaming_endpoint already established the pattern: shallow only
+# checks config (no network), --deep actually calls the server. TTS needs
+# its own deep probe specifically because "configured but broken" (a
+# rejected voice id, a dead credit balance) is otherwise indistinguishable
+# from "working" until speak() silently falls back to the offline voice —
+# which is exactly the bug report this was added to diagnose.
+
+@pytest.fixture(autouse=True)
+def _tts_configured(monkeypatch):
+    monkeypatch.setattr(config, "SERVER_URL", "http://test-server:5002")
+    monkeypatch.setattr(config, "API_KEY", "test-key")
+    monkeypatch.setattr(config, "SESSION_ID", "pet-test")
+    monkeypatch.setattr(config, "ELEVENLABS_VOICE_ID", "abc123voiceid")
+
+
+def test_tts_shallow_does_not_touch_the_network():
+    with patch("requests.post") as post:
+        check = doctor.check_tts(deep=False)
+    post.assert_not_called()
+    assert check.status == OK
+    assert "not probed" in check.detail
+
+
+def test_tts_deep_reports_success():
+    response = MagicMock(status_code=200, content=b"\x00\x01" * 100)
+    with patch("requests.post", return_value=response) as post:
+        check = doctor.check_tts(deep=True)
+    post.assert_called_once()
+    assert check.status == OK
+    assert "200 bytes" in check.detail
+
+    args, kwargs = post.call_args
+    assert args[0] == "http://test-server:5002/desk/tts"
+    assert kwargs["headers"] == {"X-Desk-Api-Key": "test-key"}
+    assert kwargs["json"]["voice_id"] == "abc123voiceid"
+
+
+def test_tts_deep_surfaces_the_servers_error_message():
+    """A 404 with 'voice not found' is exactly the shape of the reported bug:
+    a vb: clone owned by someone else, or a stale/mistyped id."""
+    response = MagicMock(status_code=404)
+    response.json.return_value = {"error": "voice not found"}
+    with patch("requests.post", return_value=response):
+        check = doctor.check_tts(deep=True)
+    assert check.status == FAIL
+    assert "404" in check.detail
+    assert "voice not found" in check.detail
+    assert check.fix
+
+
+def test_tts_deep_handles_a_network_failure_without_raising():
+    with patch("requests.post", side_effect=OSError("no route to host")):
+        check = doctor.check_tts(deep=True)
+    assert check.status == FAIL
+    assert "no route to host" in check.detail
+
+
+def test_tts_run_wires_deep_through_automatically(monkeypatch):
+    """run()'s TypeError-fallback dance (doctor.py's CHECKS loop) must
+    actually reach check_tts's deep parameter, not silently call it shallow."""
+    seen = []
+
+    def fake_check_tts(deep=False):
+        seen.append(deep)
+        return doctor.Check("text-to-speech", OK, "stub")
+
+    monkeypatch.setattr(doctor, "CHECKS", (("tts", fake_check_tts),))
+    doctor.run(deep=True)
+    assert seen == [True]

tests/test_lipsync.py

@@ -0,0 +1,135 @@
+"""Amplitude → mouth openness, from PCM samples to the drawn frame.
+
+Two halves, tested separately because only one of them needs a display:
+`tts.level_of`/`envelope` turn samples into a 0..1 loudness, and
+`PetWindow._mouth_frame` turns that loudness into a frame index. They meet at
+the `mouth` signal, which the pipeline smoke test covers end to end.
+"""
+
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet.audio import tts
+
+
+def frame(amplitude: int, samples: int = 480) -> np.ndarray:
+    return np.full(samples, amplitude, dtype=np.int16)
+
+
+# ── loudness ────────────────────────────────────────────────────────────────
+
+def test_silence_closes_the_mouth():
+    assert tts.level_of(np.zeros(480, dtype=np.int16)) == 0.0
+
+
+def test_a_loud_frame_opens_it_fully():
+    assert tts.level_of(frame(30000)) == 1.0
+
+
+def test_the_level_never_leaves_zero_to_one():
+    """It indexes a frame list; out of range is an IndexError on the UI thread."""
+    for amplitude in (0, 1, 500, 6000, 20000, 32767):
+        assert 0.0 <= tts.level_of(frame(amplitude)) <= 1.0
+
+
+def test_it_rises_with_amplitude():
+    quiet, middling, loud = (tts.level_of(frame(a)) for a in (800, 4000, 12000))
+    assert quiet < middling < loud
+
+
+def test_quiet_speech_still_moves_the_mouth_visibly():
+    """The sqrt curve is the whole point. Speech spends most of its time well
+    below peak, so a linear map leaves the mouth barely open for normal talking
+    and the pet looks like it's mumbling."""
+    assert tts.level_of(frame(1500)) > 0.15
+
+
+def test_an_empty_frame_is_silence_not_a_crash():
+    assert tts.level_of(np.zeros(0, dtype=np.int16)) == 0.0
+
+
+# ── the envelope of a whole clip ────────────────────────────────────────────
+
+def test_an_envelope_has_one_value_per_frame_at_the_requested_rate():
+    one_second = np.zeros(16000, dtype=np.int16)
+    assert len(tts.envelope(one_second, 16000, fps=30)) == pytest.approx(30, abs=1)
+
+
+def test_an_envelope_tracks_loud_and_quiet_stretches():
+    pcm = np.concatenate([np.zeros(8000, dtype=np.int16), frame(20000, 8000)])
+    levels = tts.envelope(pcm, 16000, fps=10)
+    assert max(levels[:4]) == 0.0      # the silent half
+    assert min(levels[-4:]) > 0.5      # the loud half
+
+
+def test_an_empty_clip_has_an_empty_envelope():
+    assert tts.envelope(np.zeros(0, dtype=np.int16), 16000) == []
+
+
+# ── the drawn frame ─────────────────────────────────────────────────────────
+
+@pytest.fixture
+def window():
+    """Needs a QApplication; run with QT_QPA_PLATFORM=offscreen."""
+    from PySide6.QtWidgets import QApplication
+
+    from bolt_pet.ui.pet_window import PetWindow
+
+    _app = QApplication.instance() or QApplication(["test"])
+    win = PetWindow()
+    yield win
+    win.close()
+
+
+class FakeAnimation:
+    """Stands in for sprite.Animation — _mouth_frame only wants `.frames`."""
+
+    def __init__(self, frames):
+        self.frames = list(frames)
+
+
+def test_the_talking_frame_follows_the_level(window):
+    """The talking frames are an openness ramp — closed first, widest last —
+    specifically so loudness can index them directly."""
+    animation = FakeAnimation(["closed", "a", "b", "c", "d", "open"])
+
+    assert window._mouth_frame(animation) is None      # no level set yet
+
+    window.set_mouth(0.0)
+    assert window._mouth_frame(animation) == "closed"
+
+    window.set_mouth(1.0)
+    assert window._mouth_frame(animation) == "open"
+
+    window.set_mouth(0.5)
+    assert window._mouth_frame(animation) not in ("closed", "open")
+
+
+def test_a_stale_level_gives_the_animation_back(window):
+    """If the audio thread stops sending levels — TTS died, the clip ended
+    without a final zero — the mouth must not freeze half-open forever. After a
+    moment it falls back to the ordinary looping animation."""
+    animation = FakeAnimation(["a", "b", "c"])
+    window.set_mouth(0.9)
+    assert window._mouth_frame(animation) is not None
+
+    window._mouth_at = time.monotonic() - 5.0
+    assert window._mouth_frame(animation) is None
+
+
+def test_a_single_frame_animation_falls_back_instead_of_indexing(window):
+    """Art with one talking frame can't lip-sync; it must not try."""
+    window.set_mouth(1.0)
+    assert window._mouth_frame(FakeAnimation(["only"])) is None
+    assert window._mouth_frame(FakeAnimation([])) is None
+
+
+def test_no_animation_at_all_is_handled(window):
+    window.set_mouth(0.5)
+    assert window._mouth_frame(None) is None

tests/test_mic.py

@@ -100,6 +100,32 @@ def test_returns_none_when_should_continue_stops_before_speech():
     assert result is None
 
 
+def test_list_input_devices_filters_to_input_capable(monkeypatch):
+    fake_devices = [
+        {"name": "Speakers", "max_input_channels": 0, "hostapi": 0},
+        {"name": "USB Mic", "max_input_channels": 2, "hostapi": 0},
+        {"name": "Line In", "max_input_channels": 1, "hostapi": 1},
+    ]
+    fake_hostapis = [{"name": "MME"}, {"name": "WASAPI"}]
+
+    class _FakeSoundDevice:
+        @staticmethod
+        def query_devices():
+            return fake_devices
+
+        @staticmethod
+        def query_hostapis(index):
+            return fake_hostapis[index]
+
+    monkeypatch.setitem(sys.modules, "sounddevice", _FakeSoundDevice())
+
+    devices = mic.list_input_devices()
+    assert devices == [
+        {"index": 1, "name": "USB Mic", "hostapi": "MME"},
+        {"index": 2, "name": "Line In", "hostapi": "WASAPI"},
+    ]
+
+
 def test_pcm_to_wav_bytes_round_trips_via_wave_module():
     import wave
     import io

tests/test_pipeline_smoke.py

@@ -0,0 +1,289 @@
+"""End-to-end: microphone in, speech out, over real HTTP.
+
+Every other test in this suite injects a fake at the seam it cares about, and
+every one of them passed all week while these got through to production:
+
+  - notifications sitting unspoken for minutes (a clock stamped in the wrong
+    order, two correct units)
+  - the pet saying the same thing twice (server discarded prose, model repeated
+    it — both sides behaving as written)
+  - `[laughing]` read out loud (a tag that means something to one model and
+    nothing to the next one down the pipe)
+  - a device command written as prose (extractor fine, prompt fine, no marker)
+
+They were all *interaction* bugs. So this one runs the actual pipeline against
+a real socket: a threaded HTTP server that speaks the desk protocol, the real
+`server_client` doing real requests (including NDJSON streaming), the real
+controller loop and state machine. The only fakes are where the hardware is —
+the mic stream and the speakers — because those are the two things a test
+genuinely cannot have.
+"""
+
+import json
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+import numpy as np
+import pytest
+from PySide6.QtWidgets import QApplication
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet import controller as controller_mod
+from bolt_pet.notifications import Notification
+from bolt_pet.state import PetState
+
+_app = QApplication.instance() or QApplication(["test"])
+
+
+# ── a desk server that actually listens on a port ───────────────────────────
+
+class FakeDesk:
+    """Scripted responses, real HTTP. Set `.script` per test."""
+
+    def __init__(self):
+        self.script = {}
+        self.requests = []
+        self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler())
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+
+    @property
+    def url(self) -> str:
+        host, port = self._server.server_address[:2]
+        return f"http://{host}:{port}"
+
+    def stop(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+
+    def _handler(self):
+        desk = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def log_message(self, *_args):
+                pass  # the test output is not an access log
+
+            def do_GET(self):
+                path = self.path.split("?")[0]
+                desk.requests.append(("GET", path))
+                self._json(desk.script.get(path, {"files": []}))
+
+            def do_POST(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                body = json.loads(self.rfile.read(length) or b"{}")
+                path = self.path.split("?")[0]
+                desk.requests.append(("POST", path, body))
+                response = desk.script.get(path)
+                if callable(response):
+                    response = response(body)
+                if path.endswith("converse_stream"):
+                    self._ndjson(response or [])
+                else:
+                    self._json(response if response is not None else {"type": "reply", "text": "ok"})
+
+            def _json(self, payload):
+                data = json.dumps(payload).encode()
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+
+            def _ndjson(self, events):
+                self.send_response(200)
+                self.send_header("Content-Type", "application/x-ndjson")
+                self.end_headers()
+                for event in events:
+                    self.wfile.write((json.dumps(event) + "\n").encode())
+                    self.wfile.flush()
+
+        return Handler
+
+
+class FakeMic:
+    """Loud frames then quiet ones, so the VAD ends the utterance on its own."""
+
+    def __init__(self, loud=8, quiet=60):
+        self.frames = ([np.full((320, 1), 4000, dtype=np.int16)] * loud
+                       + [np.zeros((320, 1), dtype=np.int16)] * quiet)
+
+    def read(self, _n):
+        return (self.frames.pop(0) if self.frames
+                else np.zeros((320, 1), dtype=np.int16)), None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_a):
+        return False
+
+
+@pytest.fixture
+def pipeline(monkeypatch):
+    """A controller wired to a real local server, with fake ears and mouth."""
+    desk = FakeDesk()
+    spoken: list[str] = []
+    levels: list[float] = []
+
+    monkeypatch.setattr(controller_mod.config, "SERVER_URL", desk.url)
+    monkeypatch.setattr(controller_mod.config, "API_KEY", "test-key")
+    monkeypatch.setattr(controller_mod.config, "SESSION_ID", "pet-smoke")
+    monkeypatch.setattr(controller_mod.config, "SILENCE_END_SEC", 0.2)
+    monkeypatch.setattr(controller_mod.config, "MIN_UTTERANCE_S", 0.0)
+    monkeypatch.setattr(controller_mod.config, "RECEIVE_FILES", False)
+    # Off by default so each test picks its own path; the streaming tests opt in.
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
+    monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text)
+    monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False)
+    # Deepgram and the speakers are the two things a test can't have.
+    monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's on my disk?")
+    monkeypatch.setattr(controller_mod.stt_stream.StreamingTranscriber, "open",
+                        classmethod(lambda cls, **kw: None))
+
+    def fake_speak(text, on_error=None, should_stop=None, voice_id=None, on_level=None):
+        spoken.append(text)
+        if on_level is not None:
+            on_level(0.8)   # the mouth opens while a word plays...
+            levels.append(0.8)
+        return True
+
+    monkeypatch.setattr(controller_mod.tts, "speak", fake_speak)
+
+    ctrl = controller_mod.PetController()
+    ctrl._stream = FakeMic()
+    try:
+        yield ctrl, desk, spoken, levels
+    finally:
+        desk.stop()
+
+
+# ── the whole path ──────────────────────────────────────────────────────────
+
+def test_a_plain_turn_goes_mic_to_speaker(pipeline):
+    ctrl, desk, spoken, _levels = pipeline
+    desk.script["/desk/converse"] = {"type": "reply", "text": "About sixty percent full."}
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["About sixty percent full."]
+    assert ctrl._state.state == PetState.IDLE
+    posted = [r for r in desk.requests if r[0] == "POST"]
+    assert posted[0][1] == "/desk/converse"
+    assert "what's on my disk?" in posted[0][2]["text"]
+    assert ctrl.history.entries()[-1].text == "About sixty percent full."
+
+
+def test_a_streamed_turn_speaks_each_sentence_as_it_lands(pipeline, monkeypatch):
+    """The NDJSON is parsed by the real client over a real socket — the layer
+    that a mocked `converse` can never exercise."""
+    ctrl, desk, spoken, _levels = pipeline
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
+    desk.script["/desk/converse_stream"] = [
+        {"type": "say", "text": "The disk is fine."},
+        {"type": "say", "text": "About sixty percent used."},
+        {"type": "reply", "text": "The disk is fine. About sixty percent used.",
+         "already_spoken": True},
+    ]
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["The disk is fine.", "About sixty percent used."]
+    # The final reply must not be spoken a third time.
+    assert len(spoken) == 2
+
+
+def test_a_tool_turn_says_give_me_a_sec_then_the_answer(pipeline, monkeypatch):
+    """Holding line, relayed command, and the real answer — in that order."""
+    ctrl, desk, spoken, _levels = pipeline
+    ran = []
+    monkeypatch.setattr(controller_mod.server_client, "run_local_command",
+                        lambda cmd: ran.append(cmd) or "[exit 0]\n60% used")
+    desk.script["/desk/converse"] = {
+        "type": "command", "command": "df -h /", "token": "tok",
+        "say": "Let me check that for you.",
+    }
+    desk.script["/desk/tool_result"] = {"type": "reply", "text": "Sixty percent used."}
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["Let me check that for you.", "Sixty percent used."]
+    assert ran == ["df -h /"]
+    relayed = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/tool_result")
+    assert relayed[2]["output"].endswith("60% used")
+
+
+def test_a_device_command_never_reaches_the_shell(pipeline, monkeypatch):
+    ctrl, desk, spoken, _levels = pipeline
+    ran = []
+    monkeypatch.setattr(controller_mod.server_client, "run_local_command", lambda cmd: ran.append(cmd))
+    actions = []
+    ctrl.action.connect(actions.append)
+    desk.script["/desk/converse"] = {
+        "type": "command", "command": "petctl emote wave", "token": "tok"}
+    desk.script["/desk/tool_result"] = {"type": "reply", "text": "There you go."}
+
+    ctrl._handle_conversation_turn()
+
+    assert ran == []
+    assert actions == [{"action": "emote", "emote": "wave"}]
+    assert spoken == ["There you go."]
+
+
+def test_the_mouth_moves_with_the_audio(pipeline):
+    """Lip-sync is only real if the level actually reaches the window."""
+    ctrl, desk, _spoken, levels = pipeline
+    mouth = []
+    ctrl.mouth.connect(mouth.append)
+    desk.script["/desk/converse"] = {"type": "reply", "text": "Talking now."}
+
+    ctrl._handle_conversation_turn()
+
+    assert levels, "TTS was never given a level callback"
+    assert 0.8 in mouth              # opened while speaking...
+    assert mouth[-1] == 0.0          # ...and closed at the end
+
+
+def test_a_notification_is_forwarded_and_spoken(pipeline):
+    """The path that was silently sitting for five to ten minutes."""
+    ctrl, desk, spoken, _levels = pipeline
+    ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
+    desk.script["/desk/converse"] = {"type": "reply", "text": "Harry says he's around."}
+
+    ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
+    ctrl._maybe_heartbeat()
+
+    assert spoken == ["Harry says he's around."]
+    forwarded = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/converse")
+    assert "Harry" in forwarded[2]["text"]
+
+
+def test_streaming_falls_back_when_the_server_is_older(pipeline, monkeypatch):
+    """A server without /desk/converse_stream (or one that answers with nothing)
+    must not cost a turn — the client drops to the plain endpoint. This is how
+    the pet keeps working against a container that hasn't been updated yet."""
+    ctrl, desk, spoken, _levels = pipeline
+    monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
+    desk.script["/desk/converse_stream"] = []          # nothing streamed back
+    desk.script["/desk/converse"] = {"type": "reply", "text": "Still here."}
+
+    ctrl._handle_conversation_turn()
+
+    assert spoken == ["Still here."]
+    paths = [r[1] for r in desk.requests if r[0] == "POST"]
+    assert paths == ["/desk/converse_stream", "/desk/converse"]
+
+
+def test_a_dead_server_leaves_the_pet_usable(pipeline):
+    """It should flash an error and go back to listening, not wedge."""
+    ctrl, desk, spoken, _levels = pipeline
+    desk.stop()   # the server disappears mid-session
+    states = []
+    ctrl.state_changed.connect(states.append)
+
+    ctrl._handle_conversation_turn()
+
+    assert states[-2:] == ["error", "idle"]
+    assert spoken == []

tests/test_speech.py

@@ -0,0 +1,260 @@
+"""The rules about talking — the four kinds of utterance and the mic policy.
+
+These were four near-copies in the controller before, and the copies had
+drifted: one didn't arm barge-in, one skipped the follow-up rule. The value of
+having one `Speaker` is only real if the differences between the kinds stay
+*visible*, so this asserts on the differences rather than on the machinery.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet import config, speech
+from bolt_pet.speech import Speaker, Utterance
+from bolt_pet.state import PetState, PetStateMachine
+
+
+class FakeTts:
+    def __init__(self, completed=True):
+        self.completed = completed
+        self.calls = []
+
+    def speak(self, text, on_error=None, should_stop=None, voice_id=None, on_level=None):
+        self.calls.append({"text": text, "voice_id": voice_id,
+                           "interruptible": should_stop is not None})
+        if on_level is not None:
+            on_level(0.7)
+        return self.completed
+
+    def play_pcm(self, pcm, sample_rate, should_stop=None, on_level=None):
+        self.calls.append({"pcm": pcm, "rate": sample_rate,
+                           "interruptible": should_stop is not None})
+        return self.completed
+
+
+class FakeBargeIn:
+    def __init__(self):
+        self.resets = 0
+
+    def reset(self):
+        self.resets += 1
+
+    def check(self, _frame=None):
+        return False
+
+
+def build(**kwargs):
+    """A speaker plus the things worth asserting on."""
+    state = PetStateMachine()
+    tts = kwargs.pop("tts", None) or FakeTts()
+    said, logged, recorded, levels = [], [], [], []
+    speaker = Speaker(
+        state=state, tts=tts,
+        history=recorded.append,
+        on_said=said.append,
+        on_log=logged.append,
+        on_level=levels.append,
+        **kwargs,
+    )
+    return speaker, state, tts, said, logged, recorded, levels
+
+
+# ── what distinguishes the four kinds ───────────────────────────────────────
+
+def test_a_reply_is_recorded_but_a_holding_line_is_not():
+    """Filler must not push the actual answer out of the transcript."""
+    speaker, state, _tts, _said, _logged, recorded, _levels = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.holding("Give me a sec."))
+    speaker.say(Utterance.reply("Sixty percent used."))
+
+    assert recorded == ["Sixty percent used."]
+
+
+def test_a_holding_line_resumes_the_turn_it_interrupted():
+    """The turn isn't over — a tool is still running — so it must go back to
+    THINKING, not drop to IDLE and end the turn."""
+    speaker, state, _tts, *_ = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.holding("Let me check."))
+
+    assert state.state == PetState.THINKING
+
+
+def test_a_holding_line_cannot_be_talked_over():
+    """Cutting off "give me a sec" strands the tool that's already running."""
+    detector = FakeBargeIn()
+    speaker, state, tts, *_ = build(barge_in=lambda: detector)
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.holding("One sec."))
+    assert tts.calls[-1]["interruptible"] is False
+
+    speaker.say(Utterance.reply("Done."))
+    assert tts.calls[-1]["interruptible"] is True
+
+
+def test_a_streamed_sentence_stays_talking_between_sentences():
+    """Otherwise the sprite flickers idle-talking-idle down a long answer."""
+    speaker, state, *_ = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.stream_chunk("The disk is fine."))
+
+    assert state.state == PetState.TALKING
+
+
+def test_a_scene_is_recorded_because_the_user_heard_it():
+    speaker, state, _tts, _said, _logged, recorded, _levels = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say_pcm(Utterance.scene("Once upon a time."), pcm=b"\x00\x00", sample_rate=44100)
+
+    assert recorded == ["Once upon a time."]
+    assert state.state == PetState.THINKING
+
+
+# ── barge-in ordering, which is what the copies got wrong ───────────────────
+
+def test_the_detector_is_reset_after_playback_not_only_before():
+    """Playback fed the pet's own voice into the wake model's window. If it
+    isn't cleared afterwards, the idle listener re-hears the last sentence and
+    the pet answers itself."""
+    detector = FakeBargeIn()
+    speaker, state, *_ = build(barge_in=lambda: detector)
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("Hello there."))
+
+    assert detector.resets == 2  # armed before, cleared after
+
+
+def test_the_barge_in_detail_is_captured_before_the_reset():
+    """Read it after the reset and every interruption reports zeroed counters —
+    which reads like hard evidence and is nothing of the sort."""
+    detector = FakeBargeIn()
+    details = ["score 0.81 at frame 12", "score 0.000 at frame 0"]
+    speaker, state, *_ = build(barge_in=lambda: detector,
+                               detail_of=lambda: details.pop(0))
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("Hello there."))
+
+    assert speaker.last_detail == "score 0.81 at frame 12"
+
+
+def test_a_detector_that_appears_late_is_still_used():
+    """The detector is built after the speaker — it needs the mic stream — so
+    it's read through a callable. Holding a copy is how the two drift apart."""
+    detector = None
+    speaker, state, tts, *_ = build(barge_in=lambda: detector)
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("Before."))
+    assert tts.calls[-1]["interruptible"] is False
+
+    detector = FakeBargeIn()
+    speaker.say(Utterance.reply("After."))
+    assert tts.calls[-1]["interruptible"] is True
+
+
+# ── the mouth ───────────────────────────────────────────────────────────────
+
+def test_the_mouth_is_closed_when_the_line_ends():
+    """A pet left mid-vowel after the audio stops looks broken."""
+    speaker, state, _tts, _said, _logged, _recorded, levels = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("Talking."))
+
+    assert levels[0] > 0
+    assert levels[-1] == 0.0
+
+
+# ── text handling ───────────────────────────────────────────────────────────
+
+def test_an_empty_line_is_not_spoken_at_all():
+    """A reply that is nothing but an audio tag reduces to '' — and a silent
+    bubble with no audio is better than the pet announcing "laughing"."""
+    speaker, state, tts, said, *_ = build()
+
+    assert speaker.say(Utterance.reply("[laughing]")) is True
+    assert tts.calls == []
+    assert said == []
+    assert state.state == PetState.IDLE
+
+
+def test_the_bubble_gets_display_text_and_tts_gets_the_original():
+    """The bubble keeps emoji and drops markdown; TTS does its own stripping
+    (inside speak(), so every path is covered) and needs the real text."""
+    speaker, state, tts, said, *_ = build()
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("**Sixty** percent 🎉"))
+
+    assert said == ["Sixty percent 🎉"]
+    assert tts.calls[-1]["text"] == "**Sixty** percent 🎉"
+
+
+def test_a_voice_override_reaches_tts():
+    speaker, state, tts, *_ = build(voice_id=lambda: "voice-abc")
+    state.transition(PetState.LISTENING)
+    state.transition(PetState.THINKING)
+
+    speaker.say(Utterance.reply("In character."))
+
+    assert tts.calls[-1]["voice_id"] == "voice-abc"
+
+
+# ── the follow-up rule ──────────────────────────────────────────────────────
+
+@pytest.fixture
+def follow_ups_on(monkeypatch):
+    monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", True)
+    monkeypatch.setattr(config, "FOLLOW_UP_MAX_TURNS", 3)
+
+
+def test_an_interruption_always_reopens_the_mic(follow_ups_on):
+    """You talked over it — you are mid-sentence, so it has to listen."""
+    keep, why = speech.follow_up_decision("Anything else?", completed=False, follow_ups=99)
+    assert keep is True
+    assert why == "interrupted"
+
+
+def test_a_question_keeps_the_mic_open_without_the_wake_word(follow_ups_on):
+    keep, why = speech.follow_up_decision("Want me to check?", completed=True, follow_ups=0)
+    assert (keep, why) == (True, "question")
+
+
+def test_a_statement_ends_the_turn(follow_ups_on):
+    keep, _why = speech.follow_up_decision("Sixty percent used.", completed=True, follow_ups=0)
+    assert keep is False
+
+
+def test_the_cap_stops_a_server_that_ends_every_reply_with_a_question(follow_ups_on):
+    """Otherwise mic noise loops it forever."""
+    assert speech.follow_up_decision("Ok?", completed=True, follow_ups=2)[0] is True
+    keep, why = speech.follow_up_decision("Ok?", completed=True, follow_ups=3)
+    assert keep is False
+    assert "cap" in why
+
+
+def test_the_rule_can_be_switched_off(monkeypatch):
+    monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", False)
+    assert speech.follow_up_decision("Ok?", completed=True, follow_ups=0)[0] is False

tests/test_speech_text.py

@@ -4,6 +4,8 @@
 import sys
 from pathlib import Path
 
+import pytest
+
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
 from bolt_pet.speech_text import for_display, for_speech, is_question
@@ -77,3 +79,33 @@ def test_is_question_ignores_question_marks_that_are_not_spoken():
     assert not is_question("Docs are at https://example.com/x?y=1")
     assert not is_question("")
     assert not is_question(None)
+
+
+# ── ElevenLabs v3 delivery tags (observed live 2026-07-31) ──────────────────
+# "[laughing] That one came through clean" was spoken as "laughing That one
+# came through clean": the tags mean something to the dialogue model inside a
+# dialoguectl scene, and nothing at all to the ordinary reply voice.
+
+def test_delivery_tags_are_never_spoken_aloud():
+    spoken = for_speech("[laughing] That one came through clean.")
+    assert "laughing" not in spoken
+    assert spoken.startswith("That one came through clean")
+
+
+def test_the_bubble_does_not_caption_a_laugh_nobody_heard():
+    assert "[laughing]" not in for_display("[laughing] All good.")
+
+
+@pytest.mark.parametrize("tag", [
+    "[whispering]", "[cheerfully]", "[sighs]", "[laughs]", "[clears throat]",
+    "[nervously]", "[excited]", "[pause]", "[shouting]",
+])
+def test_the_common_tags_are_all_covered(tag):
+    assert "" == for_speech(tag).strip()
+
+
+def test_ordinary_bracketed_text_survives():
+    """The tags are matched narrowly on purpose — real bracketed content is
+    part of what the user asked to hear."""
+    assert "1" in for_speech("See reference [1] for details.")
+    assert "docs" in for_speech("It's in [the docs] somewhere.")

tests/test_stt.py

@@ -0,0 +1,103 @@
+"""stt.transcribe() — the "guaranteed" one-shot fallback.
+
+There is no separate REST endpoint server-side any more: /desk/stt is a
+websocket relay only, so this connects the exact same way stt_stream.py's
+opportunistic streaming path does (via stt_stream.connect(), unconditionally
+— not gated by STT_STREAMING, since there's nothing left to fall back to).
+"""
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet.audio import stt, stt_stream
+
+
+class FakeSocket:
+    def __init__(self, messages=(), fail_to_connect=False):
+        self.sent = []
+        self.closed = False
+        self._fail_to_connect = fail_to_connect
+        self._messages = list(messages)
+
+    def send_binary(self, data):
+        self.sent.append(data)
+
+    def send(self, text):
+        self.sent.append(text)
+
+    def recv(self):
+        if self._messages:
+            return self._messages.pop(0)
+        time.sleep(0.01)
+        raise ConnectionError("closed")
+
+    def close(self):
+        self.closed = True
+
+
+def _results(transcript, is_final=True):
+    return json.dumps({
+        "type": "Results", "is_final": is_final,
+        "channel": {"alternatives": [{"transcript": transcript}]},
+    })
+
+
+@pytest.fixture(autouse=True)
+def _configured(monkeypatch):
+    monkeypatch.setattr(stt.config, "SERVER_URL", "http://test-server:5002")
+    monkeypatch.setattr(stt.config, "API_KEY", "test-key")
+
+
+def test_transcribe_raises_when_not_configured(monkeypatch):
+    monkeypatch.setattr(stt.config, "API_KEY", "")
+    with pytest.raises(stt.SttError, match="BOLT_SERVER_URL"):
+        stt.transcribe(np.zeros(320, dtype=np.int16))
+
+
+def test_transcribe_connects_and_feeds_the_whole_utterance(monkeypatch):
+    socket = FakeSocket([_results("turn on the lights")])
+    monkeypatch.setattr(stt_stream, "connect", lambda: socket)
+
+    pcm = np.full(3200, 500, dtype=np.int16)
+    text = stt.transcribe(pcm)
+
+    assert text == "turn on the lights"
+    # one binary frame (the whole utterance) plus the close message
+    assert len(socket.sent) == 2
+    assert socket.closed
+
+
+def test_transcribe_raises_when_the_server_is_unreachable(monkeypatch):
+    def refuse():
+        raise OSError("no route to host")
+
+    monkeypatch.setattr(stt_stream, "connect", refuse)
+
+    with pytest.raises(stt.SttError, match="couldn't reach"):
+        stt.transcribe(np.zeros(320, dtype=np.int16))
+
+
+def test_transcribe_returns_empty_string_for_silence_not_an_error(monkeypatch):
+    """No speech recognized is a legitimate outcome, not a failure — callers
+    (controller.py) treat "" as "say nothing" rather than logging an error."""
+    socket = FakeSocket([])  # never says anything back
+    monkeypatch.setattr(stt_stream, "connect", lambda: socket)
+
+    assert stt.transcribe(np.zeros(320, dtype=np.int16)) == ""
+
+
+def test_transcribe_is_unaffected_by_stt_streaming_being_off(monkeypatch):
+    """The opportunistic accelerator and the guaranteed fallback share a
+    connector, but STT_STREAMING must only gate the former."""
+    monkeypatch.setattr(stt.config, "STT_STREAMING", False)
+    socket = FakeSocket([_results("still works")])
+    monkeypatch.setattr(stt_stream, "connect", lambda: socket)
+
+    assert stt.transcribe(np.zeros(320, dtype=np.int16)) == "still works"

tests/test_stt_stream.py

@@ -0,0 +1,224 @@
+"""Streaming speech-to-text: the protocol, and the fallback that makes it safe
+to switch on at all.
+
+A fake websocket throughout — no network, no Deepgram account, no audio.
+"""
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet.audio import mic
+from bolt_pet.audio.stt_stream import StreamingTranscriber
+
+
+class FakeSocket:
+    """Records what was sent; replays scripted Deepgram frames."""
+
+    def __init__(self, messages=(), fail_on_send=False):
+        self.sent = []
+        self.closed = False
+        self.fail_on_send = fail_on_send
+        self._messages = list(messages)
+
+    def send_binary(self, data):
+        if self.fail_on_send:
+            raise ConnectionError("socket died")
+        self.sent.append(data)
+
+    def send(self, text):
+        self.sent.append(text)
+
+    def recv(self):
+        if self._messages:
+            return self._messages.pop(0)
+        time.sleep(0.01)
+        raise ConnectionError("closed")
+
+    def close(self):
+        self.closed = True
+
+
+def _results(transcript, is_final=True):
+    return json.dumps({
+        "type": "Results", "is_final": is_final,
+        "channel": {"alternatives": [{"transcript": transcript}]},
+    })
+
+
+def _frame(value=1000):
+    return np.full(320, value, dtype=np.int16)
+
+
+# ── the protocol ────────────────────────────────────────────────────────────
+
+def test_frames_go_up_as_they_are_captured():
+    socket = FakeSocket([_results("what's the weather")])
+    session = StreamingTranscriber(socket)
+
+    for _ in range(3):
+        session.feed(_frame())
+    text = session.finish()
+
+    assert len(socket.sent) == 4          # three frames plus the close message
+    assert text == "what's the weather"
+    assert socket.closed
+
+
+def test_only_final_results_are_kept():
+    """Interim hypotheses change under you; concatenating them would produce
+    "what what's what's the what's the weather"."""
+    socket = FakeSocket([
+        _results("what's", is_final=False),
+        _results("what's the", is_final=False),
+        _results("what's the weather", is_final=True),
+    ])
+    session = StreamingTranscriber(socket)
+    assert session.finish() == "what's the weather"
+
+
+def test_several_final_segments_are_joined():
+    socket = FakeSocket([_results("turn the lights on"), _results("in the kitchen")])
+    session = StreamingTranscriber(socket)
+    assert session.finish() == "turn the lights on in the kitchen"
+
+
+def test_junk_frames_are_ignored_rather_than_killing_the_reader():
+    """An exception on the reader thread would silently end transcription for
+    the rest of the utterance."""
+    socket = FakeSocket(["not json at all", '{"type":"Metadata"}',
+                         _results("still works")])
+    session = StreamingTranscriber(socket)
+    assert session.finish() == "still works"
+
+
+def test_an_empty_frame_means_the_socket_closed():
+    """websocket-client returns "" from recv() on a closed connection, so it
+    ends the read loop rather than being treated as a blank transcript."""
+    socket = FakeSocket([_results("heard this much"), "", _results("never arrives")])
+    session = StreamingTranscriber(socket)
+    assert session.finish() == "heard this much"
+
+
+def test_a_socket_that_dies_mid_utterance_gives_up_quietly():
+    socket = FakeSocket([], fail_on_send=True)
+    session = StreamingTranscriber(socket)
+
+    session.feed(_frame())      # must not raise — recording carries on
+    assert session.finish() == ""
+
+
+# ── opening: failure is an ordinary outcome ────────────────────────────────
+
+def test_open_returns_none_when_it_cannot_connect():
+    """None means "the one-shot path will do it", not an error."""
+    def refuse():
+        raise OSError("no network")
+
+    assert StreamingTranscriber.open(connect=refuse) is None
+
+
+def test_open_returns_a_session_when_it_can():
+    session = StreamingTranscriber.open(connect=lambda: FakeSocket([_results("hi")]))
+    assert session is not None
+    assert session.finish() == "hi"
+
+
+def test_streaming_is_off_without_the_switch_or_server_config(monkeypatch):
+    from bolt_pet.audio import stt_stream
+
+    monkeypatch.setattr(stt_stream.config, "STT_STREAMING", False)
+    monkeypatch.setattr(stt_stream.config, "SERVER_URL", "http://test-server:5002")
+    monkeypatch.setattr(stt_stream.config, "API_KEY", "test-key")
+    assert stt_stream.available() is False
+    monkeypatch.setattr(stt_stream.config, "STT_STREAMING", True)
+    assert stt_stream.available() is True
+    monkeypatch.setattr(stt_stream.config, "API_KEY", "")
+    assert stt_stream.available() is False
+
+
+# ── connecting to the server relay ──────────────────────────────────────────
+
+def test_connect_builds_the_server_relay_url(monkeypatch):
+    """No local Deepgram account any more — the pet connects to its own
+    server's /desk/stt, authenticated with its own desk key."""
+    from bolt_pet.audio import stt_stream
+
+    monkeypatch.setattr(stt_stream.config, "SERVER_URL", "http://my-server:5002")
+    monkeypatch.setattr(stt_stream.config, "API_KEY", "my-desk-key")
+    monkeypatch.setattr(stt_stream.config, "SESSION_ID", "pet-test")
+
+    captured = {}
+
+    class _FakeWebsocketModule:
+        @staticmethod
+        def create_connection(url, header=None, timeout=None):
+            captured["url"] = url
+            captured["header"] = header
+            return "a-socket"
+
+    monkeypatch.setitem(sys.modules, "websocket", _FakeWebsocketModule())
+
+    result = stt_stream.connect(sample_rate=16000)
+
+    assert result == "a-socket"
+    assert captured["url"] == (
+        "ws://my-server:5002/desk/stt?session_id=pet-test"
+        "&encoding=linear16&sample_rate=16000"
+    )
+    assert captured["header"] == ["X-Desk-Api-Key: my-desk-key"]
+
+
+def test_open_uses_connect_by_default(monkeypatch):
+    """StreamingTranscriber.open() with no injected connect() goes through
+    the real server-relay connector."""
+    from bolt_pet.audio import stt_stream
+
+    monkeypatch.setattr(stt_stream.config, "STT_STREAMING", True)
+    monkeypatch.setattr(stt_stream.config, "SERVER_URL", "http://my-server:5002")
+    monkeypatch.setattr(stt_stream.config, "API_KEY", "my-desk-key")
+    monkeypatch.setattr(stt_stream, "_connect", lambda rate: FakeSocket([_results("hi")]))
+
+    session = stt_stream.StreamingTranscriber.open()
+
+    assert session is not None
+    assert session.finish() == "hi"
+
+
+# ── the capture hook ────────────────────────────────────────────────────────
+
+class _Stream:
+    """Loud frames, then quiet ones, so the VAD ends the utterance."""
+
+    def __init__(self, loud=6, quiet=40):
+        self.frames = ([np.full((320, 1), 3000, dtype=np.int16)] * loud
+                       + [np.zeros((320, 1), dtype=np.int16)] * quiet)
+
+    def read(self, n):
+        return (self.frames.pop(0) if self.frames
+                else np.zeros((320, 1), dtype=np.int16)), None
+
+
+def test_recording_hands_every_speech_frame_to_the_listener():
+    seen = []
+    pcm = mic.record_utterance(_Stream(), on_frame=seen.append,
+                               silence_end_sec=0.2, min_utterance_s=0.0)
+    assert pcm is not None
+    assert len(seen) >= 6            # every frame of speech was streamed
+
+
+def test_a_listener_that_throws_cannot_break_the_recording():
+    """The fallback is about to need this audio — a dead stream must not cost
+    the recording too."""
+    def explode(frame):
+        raise RuntimeError("stream died")
+
+    pcm = mic.record_utterance(_Stream(), on_frame=explode,
+                               silence_end_sec=0.2, min_utterance_s=0.0)
+    assert pcm is not None and len(pcm) > 0

tests/test_tts_stream.py

@@ -3,13 +3,16 @@ no network, no audio device, no ONNX model."""
 
 import sys
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import numpy as np
+import pytest
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
 from bolt_pet import config as tts_config
-from bolt_pet.audio.tts import chunks_to_int16, model_for, voice_for
+from bolt_pet.audio import tts
+from bolt_pet.audio.tts import TtsError, chunks_to_int16, voice_for
 from bolt_pet.audio.wake_word import NearMissLog
 
 
@@ -83,7 +86,7 @@ def test_clear_resets_peak_and_entries():
     assert log.entries() == [] and log.peak == 0.0
 
 
-# ── voice / model selection (server speak_as) ───────────────────────────────
+# ── voice selection (server speak_as) ───────────────────────────────────────
 
 def test_the_override_voice_wins_over_the_configured_one(monkeypatch):
     monkeypatch.setattr(tts_config, "ELEVENLABS_VOICE_ID", "DEFAULT")
@@ -92,16 +95,81 @@ def test_the_override_voice_wins_over_the_configured_one(monkeypatch):
     assert voice_for(None) == "DEFAULT"
 
 
-def test_english_replies_in_the_default_voice_use_the_default_model(monkeypatch):
+# ── server-hosted synthesis (/desk/tts) ─────────────────────────────────────
-    monkeypatch.setattr(tts_config, "ELEVENLABS_MODEL_ID", "eleven_flash_v2")
+# No local ElevenLabs account: both the whole-clip and streaming paths post
-    monkeypatch.setattr(tts_config, "ELEVENLABS_MULTILINGUAL_MODEL_ID", "eleven_flash_v2_5")
+# to this pet's own server, same as server_client.py's other endpoints.
-    assert model_for("all good here", None) == "eleven_flash_v2"
+
+@pytest.fixture(autouse=True)
+def _configured(monkeypatch):
+    monkeypatch.setattr(tts_config, "SERVER_URL", "http://test-server:5002")
+    monkeypatch.setattr(tts_config, "API_KEY", "test-key")
+    monkeypatch.setattr(tts_config, "SESSION_ID", "pet-test")
+    monkeypatch.setattr(tts_config, "ELEVENLABS_VOICE_ID", "default-voice")
 
 
-def test_a_picked_voice_or_non_english_text_uses_the_multilingual_model(monkeypatch):
+def _mock_response(content=b"", ok=True):
-    # eleven_flash_v2 is English-only: it would read either of these as
+    resp = MagicMock()
-    # mangled phonetic English rather than failing outright.
+    resp.content = content
-    monkeypatch.setattr(tts_config, "ELEVENLABS_MODEL_ID", "eleven_flash_v2")
+    resp.raise_for_status = MagicMock() if ok else MagicMock(side_effect=Exception("boom"))
-    monkeypatch.setattr(tts_config, "ELEVENLABS_MULTILINGUAL_MODEL_ID", "eleven_flash_v2_5")
+    return resp
-    assert model_for("all good here", "VOICE1") == "eleven_flash_v2_5"
+
-    assert model_for("こんにちは", None) == "eleven_flash_v2_5"
+
+def test_synthesize_pcm_posts_to_the_servers_tts_endpoint():
+    pcm_bytes = np.array([1, 2, 3], dtype=np.int16).tobytes()
+    with patch.object(tts.requests, "post") as post:
+        post.return_value = _mock_response(pcm_bytes)
+        pcm, rate = tts.synthesize_pcm("hello there")
+    post.assert_called_once()
+    args, kwargs = post.call_args
+    assert args[0] == "http://test-server:5002/desk/tts"
+    assert kwargs["headers"] == {"X-Desk-Api-Key": "test-key"}
+    assert kwargs["json"] == {
+        "session_id": "pet-test", "text": "hello there", "voice_id": "default-voice",
+    }
+    assert pcm.tolist() == [1, 2, 3]
+    assert rate == tts_config.TTS_SAMPLE_RATE
+
+
+def test_synthesize_pcm_uses_the_override_voice():
+    with patch.object(tts.requests, "post") as post:
+        post.return_value = _mock_response(b"\x01\x00")
+        tts.synthesize_pcm("hi", voice_id="picked-voice")
+    assert post.call_args.kwargs["json"]["voice_id"] == "picked-voice"
+
+
+def test_synthesize_pcm_raises_when_not_configured(monkeypatch):
+    monkeypatch.setattr(tts_config, "API_KEY", "")
+    with pytest.raises(TtsError, match="BOLT_SERVER_URL"):
+        tts.synthesize_pcm("hi")
+
+
+def test_synthesize_pcm_raises_on_empty_audio():
+    with patch.object(tts.requests, "post") as post:
+        post.return_value = _mock_response(b"")
+        with pytest.raises(TtsError, match="no audio"):
+            tts.synthesize_pcm("hi")
+
+
+def test_synthesize_pcm_raises_when_the_request_fails():
+    with patch.object(tts.requests, "post") as post:
+        post.return_value = _mock_response(b"", ok=False)
+        with pytest.raises(TtsError, match="server tts request failed"):
+            tts.synthesize_pcm("hi")
+
+
+def test_stream_pcm_posts_to_the_same_endpoint_with_stream_true():
+    with patch.object(tts.requests, "post") as post:
+        response = MagicMock()
+        response.raise_for_status = MagicMock()
+        response.iter_content.return_value = [np.array([4, 5], dtype=np.int16).tobytes()]
+        post.return_value = response
+        chunks = list(tts.stream_pcm("hi"))
+    assert post.call_args.kwargs["stream"] is True
+    assert post.call_args[0][0] == "http://test-server:5002/desk/tts"
+    assert np.concatenate(chunks).tolist() == [4, 5]
+
+
+def test_stream_pcm_raises_when_not_configured(monkeypatch):
+    monkeypatch.setattr(tts_config, "SERVER_URL", "")
+    with pytest.raises(TtsError, match="BOLT_SERVER_URL"):
+        list(tts.stream_pcm("hi"))

tests/test_window_state.py

@@ -0,0 +1,84 @@
+"""Remembering where the pet was left — and refusing to when that's a trap."""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bolt_pet import window_state
+
+
+def test_a_saved_position_comes_back(tmp_path):
+    path = tmp_path / "window.json"
+    window_state.save(1200, 640, path)
+    assert window_state.load(path) == (1200, 640)
+
+
+def test_no_file_yet_is_not_an_error(tmp_path):
+    assert window_state.load(tmp_path / "nope.json") is None
+
+
+def test_a_corrupt_file_falls_back_to_the_default_corner(tmp_path):
+    """Whatever is in there, the pet still has to start."""
+    path = tmp_path / "window.json"
+    for junk in ("", "{", "null", "[]", '{"x": "left"}', '{"y": 3}'):
+        path.write_text(junk)
+        assert window_state.load(path) is None
+
+
+def test_saving_creates_the_cache_directory(tmp_path):
+    path = tmp_path / "deep" / "nested" / "window.json"
+    window_state.save(10, 20, path)
+    assert window_state.load(path) == (10, 20)
+
+
+def test_an_unwritable_location_is_swallowed(tmp_path):
+    """A read-only cache dir is a reason to forget the position, not to crash
+    on every drag."""
+    window_state.save(1, 2, tmp_path / "no" / "\0bad" / "window.json")
+
+
+def test_the_write_is_atomic_and_leaves_no_litter(tmp_path):
+    path = tmp_path / "window.json"
+    window_state.save(5, 5, path)
+    window_state.save(7, 7, path)
+    assert json.loads(path.read_text()) == {"x": 7, "y": 7}
+    assert [p.name for p in tmp_path.iterdir()] == ["window.json"]
+
+
+# ── validating against the screens that exist *now* ─────────────────────────
+
+LAPTOP = (0, 0, 1920, 1080)
+EXTERNAL = (1920, 0, 4480, 1440)
+
+
+def test_a_position_on_a_connected_screen_is_kept():
+    assert window_state.is_visible_on(1700, 900, 128, [LAPTOP])
+
+
+def test_a_position_on_an_unplugged_monitor_is_refused():
+    """The dangerous case: restoring it faithfully puts the pet somewhere you
+    can't see or reach."""
+    assert not window_state.is_visible_on(3000, 700, 128, [LAPTOP])
+    assert window_state.is_visible_on(3000, 700, 128, [LAPTOP, EXTERNAL])
+
+
+def test_mostly_off_screen_counts_as_gone():
+    """A few pixels of ear poking onto the desktop is not "reachable"."""
+    assert not window_state.is_visible_on(1910, 500, 128, [LAPTOP])
+    assert window_state.is_visible_on(1830, 500, 128, [LAPTOP])
+
+
+def test_touching_an_edge_is_not_overlapping():
+    assert not window_state.is_visible_on(1920, 0, 128, [LAPTOP])
+
+
+def test_negative_coordinates_are_fine_when_a_screen_is_there():
+    """Monitors left of or above the primary have negative origins."""
+    left_of_primary = (-1920, 0, 0, 1080)
+    assert window_state.is_visible_on(-900, 400, 128, [left_of_primary, LAPTOP])
+
+
+def test_no_screens_at_all_is_not_visible():
+    assert not window_state.is_visible_on(100, 100, 128, [])