Upload

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

bolt_pet/audio/barge_in.py

@@ -0,0 +1,62 @@
+"""Barge-in: notice that the user started talking *while the pet is talking*
+so playback can be cut short mid-sentence.
+
+Deliberately dumber than the utterance VAD in mic.py. The mic hears the pet's
+own voice coming back out of the speakers, so a single loud frame proves
+nothing — this requires several consecutive frames well above the normal
+speech threshold (BARGE_IN_RMS_THRESHOLD defaults to 4x VAD_RMS_THRESHOLD).
+Takes the same injectable stream shape as mic.record_utterance, so tests feed
+it fake frames instead of real audio hardware.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from .. import config
+from .mic import AudioStream, rms
+
+
+class BargeInDetector:
+    """Poll-driven: call check() repeatedly while audio plays. Each call
+    consumes exactly one mic frame (80ms at the default frame length), which
+    is also what paces the playback loop's polling."""
+
+    def __init__(
+        self,
+        stream: AudioStream,
+        threshold: int = None,
+        required_frames: int = None,
+        frame_len: int = config.FRAME_LEN,
+    ):
+        self._stream = stream
+        self._threshold = config.BARGE_IN_RMS_THRESHOLD if threshold is None else threshold
+        self._required = max(1, config.BARGE_IN_FRAMES if required_frames is None else required_frames)
+        self._frame_len = frame_len
+        self._loud_frames = 0
+
+    @property
+    def loud_frames(self) -> int:
+        return self._loud_frames
+
+    def reset(self) -> None:
+        self._loud_frames = 0
+
+    def check(self) -> bool:
+        """True once the user has been loud for long enough to count as an
+        interruption. Never raises: a mic hiccup mid-playback should not kill
+        the reply, it should just mean "no barge-in this frame"."""
+        try:
+            chunk, _ = self._stream.read(self._frame_len)
+        except Exception:
+            return False
+        frame = np.asarray(chunk)
+        if frame.ndim > 1:
+            frame = frame[:, 0]
+        if frame.size == 0:
+            return False
+        if rms(frame) >= self._threshold:
+            self._loud_frames += 1
+        else:
+            self._loud_frames = 0  # a single thump/cough shouldn't count
+        return self._loud_frames >= self._required

bolt_pet/audio/mic.py

@@ -0,0 +1,111 @@
+"""Mic capture + simple energy-based VAD utterance recording.
+
+Ported from desk_client/bolt_desk.py's record_utterance() — same tuning
+knobs, same behavior. Kept independent of any UI/threading model so it can
+be unit tested by feeding it a fake "stream" object.
+"""
+
+from __future__ import annotations
+
+import io
+import wave
+from typing import Optional, Protocol
+
+import numpy as np
+
+from .. import config
+
+
+class AudioStream(Protocol):
+    """Minimal shape of the object record_utterance() needs — matches
+    sounddevice.InputStream's .read(frames) -> (data, overflowed)."""
+
+    def read(self, frames: int): ...
+
+
+def pcm_to_wav_bytes(pcm: np.ndarray, sample_rate: int = config.SAMPLE_RATE) -> bytes:
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(pcm.tobytes())
+    return buf.getvalue()
+
+
+def rms(frame: np.ndarray) -> float:
+    return float(np.sqrt(np.mean(frame.astype(np.float64) ** 2)))
+
+
+def record_utterance(
+    stream: AudioStream,
+    should_continue=lambda: True,
+    rms_threshold: int = None,
+    silence_end_sec: float = None,
+    max_utterance_s: float = None,
+    min_utterance_s: float = None,
+    frame_len: int = config.FRAME_LEN,
+    sample_rate: int = config.SAMPLE_RATE,
+) -> Optional[np.ndarray]:
+    """Capture one utterance from *stream*: wait for speech to start, stop
+    after trailing silence. Returns None if nothing usable was heard.
+
+    *should_continue* is polled each frame so a caller can cancel recording
+    (e.g. the pet window was closed) without needing threading primitives
+    baked into this function.
+    """
+    rms_threshold = config.RMS_THRESHOLD if rms_threshold is None else rms_threshold
+    silence_end_sec = config.SILENCE_END_SEC if silence_end_sec is None else silence_end_sec
+    max_utterance_s = config.MAX_UTTERANCE_S if max_utterance_s is None else max_utterance_s
+    min_utterance_s = config.MIN_UTTERANCE_S if min_utterance_s is None else min_utterance_s
+
+    frames: list[np.ndarray] = []
+    started = False
+    silence_frames = 0
+    silence_limit = int(silence_end_sec * sample_rate / frame_len)
+    max_frames = int(max_utterance_s * sample_rate / frame_len)
+    grace_frames = int(4.0 * sample_rate / frame_len)  # wait up to 4s for speech to begin
+    waited = 0
+
+    while should_continue():
+        chunk, _ = stream.read(frame_len)
+        frame = np.asarray(chunk)[:, 0].copy()
+        frame_rms = rms(frame)
+        if not started:
+            waited += 1
+            if frame_rms >= rms_threshold:
+                started = True
+                frames.append(frame)
+            elif waited > grace_frames:
+                return None  # woke it up but said nothing
+            continue
+        frames.append(frame)
+        if frame_rms < rms_threshold:
+            silence_frames += 1
+            if silence_frames >= silence_limit:
+                break
+        else:
+            silence_frames = 0
+        if len(frames) >= max_frames:
+            break
+
+    if not frames:
+        return None
+    pcm = np.concatenate(frames)
+    if len(pcm) < min_utterance_s * sample_rate:
+        return None
+    return pcm
+
+
+def open_input_stream():
+    """Real sounddevice input stream, imported lazily so pure-logic tests
+    (record_utterance with a fake stream) don't need PortAudio installed."""
+    import sounddevice as sd
+
+    return sd.InputStream(
+        samplerate=config.SAMPLE_RATE,
+        channels=1,
+        dtype="int16",
+        blocksize=config.FRAME_LEN,
+        device=config.MIC_DEVICE,
+    )

bolt_pet/audio/stt.py

@@ -0,0 +1,42 @@
+"""Speech-to-text for the actual query, after the wake word fires.
+
+Deepgram, same as desk_client/bolt_desk.py.
+"""
+
+from __future__ import annotations
+
+import requests
+
+from .. import config
+from .mic import pcm_to_wav_bytes
+
+
+class SttError(Exception):
+    pass
+
+
+def transcribe(pcm) -> str:
+    if not config.DEEPGRAM_API_KEY:
+        raise SttError("DEEPGRAM_API_KEY is not set")
+    try:
+        response = requests.post(
+            "https://api.deepgram.com/v1/listen",
+            params={"model": config.DEEPGRAM_MODEL, "language": "en", "smart_format": "true"},
+            headers={
+                "Authorization": f"Token {config.DEEPGRAM_API_KEY}",
+                "Content-Type": "audio/wav",
+            },
+            data=pcm_to_wav_bytes(pcm),
+            timeout=30,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise SttError(f"transcription request failed: {exc}") from exc
+    try:
+        return (
+            response.json()
+            .get("results", {}).get("channels", [{}])[0]
+            .get("alternatives", [{}])[0].get("transcript", "")
+        ).strip()
+    except Exception as exc:
+        raise SttError(f"couldn't parse transcription response: {exc}") from exc

bolt_pet/audio/tts.py

@@ -0,0 +1,163 @@
+"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just
+sounddevice — no external player binary (mpv/ffplay), unlike
+desk_client/bolt_desk.py which shells out because it only targets Linux.
+
+Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
+on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
+fails, so the pet can still talk with zero cloud config.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Iterator
+
+import numpy as np
+import requests
+
+from .. import config, speech_text
+
+
+class TtsError(Exception):
+    pass
+
+
+def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
+    """Returns (pcm_int16_mono, sample_rate). Raises TtsError on failure —
+    callers should fall back to speak_offline() rather than treating this
+    as fatal."""
+    if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
+        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+    try:
+        response = requests.post(
+            f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}",
+            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
+            timeout=60,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise TtsError(f"ElevenLabs request failed: {exc}") from exc
+    pcm = np.frombuffer(response.content, dtype=np.int16)
+    if pcm.size == 0:
+        raise TtsError("ElevenLabs returned no audio")
+    return pcm, config.TTS_SAMPLE_RATE
+
+
+def stream_pcm(text: str, chunk_bytes: int = 4096) -> Iterator[np.ndarray]:
+    """Same audio as synthesize_pcm(), but yielded as it arrives from
+    ElevenLabs' /stream endpoint so playback can start on the first chunk
+    (~300ms) instead of after the whole clip is synthesized. Raises TtsError
+    before yielding anything if the request itself fails, so callers can fall
+    back cleanly; a mid-stream failure just ends the generator."""
+    if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
+        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+    try:
+        response = requests.post(
+            f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}/stream",
+            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
+            timeout=60,
+            stream=True,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc
+    return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes))
+
+
+def chunks_to_int16(byte_chunks: Iterable[bytes]) -> Iterator[np.ndarray]:
+    """Reassemble a byte stream into int16 frames. HTTP chunk boundaries fall
+    wherever they like, including *inside* a 16-bit sample, so a trailing odd
+    byte has to be carried into the next chunk — otherwise every chunk after
+    the first is shifted by one byte and plays as static."""
+    carry = b""
+    for chunk in byte_chunks:
+        if not chunk:
+            continue
+        data = carry + chunk
+        usable = len(data) - (len(data) % 2)
+        carry = data[usable:]
+        if usable:
+            yield np.frombuffer(data[:usable], dtype=np.int16)
+
+
+def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool:
+    """Play a whole clip. Returns True if it finished, False if *should_stop*
+    (barge-in) cut it short. *should_stop* is polled while audio plays — each
+    poll consumes one mic frame, which is what paces this loop."""
+    import sounddevice as sd
+
+    sd.play(pcm, samplerate=sample_rate, device=config.SPEAKER_DEVICE)
+    if not blocking:
+        return True
+    if should_stop is None:
+        sd.wait()
+        return True
+    while True:
+        try:
+            if not sd.get_stream().active:
+                break
+        except Exception:
+            break  # stream already torn down — playback is over
+        if should_stop():
+            sd.stop()
+            return False
+    return True
+
+
+def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None) -> bool:
+    """Play int16 chunks as they arrive. Returns False if interrupted."""
+    import sounddevice as sd
+
+    with sd.OutputStream(
+        samplerate=sample_rate, channels=1, dtype="int16", device=config.SPEAKER_DEVICE
+    ) as out:
+        for chunk in chunks:
+            if should_stop is not None and should_stop():
+                # abort() rather than draining: barge-in should stop the voice
+                # now, not at the end of the buffered chunk.
+                out.abort()
+                return False
+            out.write(chunk)
+    return True
+
+
+def speak_offline(text: str) -> None:
+    try:
+        import pyttsx3
+    except ImportError:
+        return  # no TTS available at all — caller already logs the text
+    engine = pyttsx3.init()
+    engine.say(text)
+    engine.runAndWait()
+
+
+def speak(text: str, on_error=None, should_stop=None) -> bool:
+    """Speak *text*, preferring streaming ElevenLabs, then whole-clip
+    ElevenLabs, then offline TTS. *on_error*, if given, is called with the
+    exception when ElevenLabs fails (useful for logging) — a fallback still
+    runs either way. Returns False if barge-in interrupted playback.
+
+    The text is sanitized first (speech_text.for_speech): server replies are
+    written for a chat window, and a voice reads markdown/emoji literally
+    ("asterisk asterisk"). Sanitizing here rather than at the call sites means
+    every path to the speakers — reply, heartbeat announcement — is covered."""
+    text = speech_text.for_speech(text)
+    if not text:
+        return True
+    if config.TTS_STREAMING:
+        try:
+            return play_stream(stream_pcm(text), config.TTS_SAMPLE_RATE, should_stop=should_stop)
+        except TtsError as exc:
+            if on_error is not None:
+                on_error(exc)
+    try:
+        pcm, sample_rate = synthesize_pcm(text)
+        return play_pcm(pcm, sample_rate, should_stop=should_stop)
+    except TtsError as exc:
+        if on_error is not None:
+            on_error(exc)
+    speak_offline(text)
+    return True

bolt_pet/audio/wake_word.py

@@ -0,0 +1,185 @@
+"""Wake-word detection via a custom-trained openWakeWord model.
+
+Uses `thunderbolt.onnx` — trained specifically for "thunderbolt", the same
+way the main repo's `desk_client/bolt_desk.py` uses `bolt.onnx` for "hey
+bolt". Same runtime (openWakeWord, ONNX inference), same per-frame
+predict()/reset() pattern; the only difference is the model file
+(WAKE_MODEL_FILE) and threshold (WAKE_WORD_THRESHOLD), both configurable via
+.env if a differently-trained model is swapped in later.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Callable, Optional, Protocol, Union
+
+import numpy as np
+
+from .. import config
+
+
+class WakeModel(Protocol):
+    def predict(self, frame: np.ndarray) -> dict: ...
+    def reset(self) -> None: ...
+
+
+class NearMissLog:
+    """Rolling record of frames that *almost* fired the wake word.
+
+    WAKE_WORD_THRESHOLD is otherwise tuned by guessing at a number in .env
+    and seeing whether the pet ignores you. Keeping the near misses (scores
+    within WAKE_NEAR_MISS_MARGIN below the threshold) turns that into
+    evidence: the tray's wake-word tuner shows what your actual "thunderbolt"
+    scores, so you can set the threshold just under it.
+
+    Pure bookkeeping — the caller supplies timestamps, so it's testable.
+    """
+
+    def __init__(self, limit: int = None, margin: float = None):
+        self._entries: deque[tuple[float, float, float]] = deque(  # (timestamp, score, threshold)
+            maxlen=max(1, config.WAKE_NEAR_MISS_LIMIT if limit is None else limit)
+        )
+        self._margin = config.WAKE_NEAR_MISS_MARGIN if margin is None else margin
+        self._peak = 0.0
+
+    @property
+    def peak(self) -> float:
+        """Highest score seen since the last reset — the "how close did I
+        get?" readout while you test the wake phrase."""
+        return self._peak
+
+    def observe(self, score: float, threshold: float, timestamp: float) -> bool:
+        """Record *score*; returns True if it counted as a near miss."""
+        self._peak = max(self._peak, score)
+        if score >= threshold or score < threshold - self._margin:
+            return False
+        self._entries.append((timestamp, score, threshold))
+        return True
+
+    def entries(self) -> list[tuple[float, float, float]]:
+        return list(self._entries)
+
+    def clear(self) -> None:
+        self._entries.clear()
+        self._peak = 0.0
+
+
+def _construct_model(model_cls, model_path: str):
+    """openwakeword's Model() constructor keyword has drifted across
+    releases (wakeword_models -> wakeword_model_paths) and some builds
+    reject inference_framework entirely — the main repo's ai/wake_word.py
+    hit the same drift and works around it the same way: try each known
+    calling convention in turn."""
+    attempts = [
+        lambda: model_cls(wakeword_model_paths=[model_path], inference_framework="onnx"),
+        lambda: model_cls(wakeword_model_paths=[model_path]),
+        lambda: model_cls(wakeword_models=[model_path], inference_framework="onnx"),
+        lambda: model_cls(wakeword_models=[model_path]),
+        lambda: model_cls([model_path]),
+    ]
+    last_exc: Optional[TypeError] = None
+    for attempt in attempts:
+        try:
+            return attempt()
+        except TypeError as exc:
+            last_exc = exc
+    raise RuntimeError(
+        f"Could not construct openwakeword.Model with any known calling convention "
+        f"(last error: {last_exc})"
+    )
+
+
+class _OpenWakeWordModel:
+    """Lazily loads the ONNX model on first use so importing this module
+    (and unit-testing listen_for_wake_word with a fake model) never requires
+    onnxruntime/openwakeword or the model file to be present."""
+
+    def __init__(self):
+        self._model = None
+
+    def _ensure_model(self):
+        if self._model is None:
+            from openwakeword.model import Model
+            # from openwakeword.utils import download_models
+
+            # # The pip package doesn't bundle its melspectrogram/embedding
+            # # feature-extraction sub-models — fetch them once on first use
+            # # (no-op if already cached in openwakeword's own resources dir).
+            # # A non-empty, non-matching model_names list keeps this from
+            # # also pulling every official pretrained wakeword model.
+            # download_models(model_names=["thunderbolt"])
+
+            self._model = _construct_model(Model, config.WAKE_MODEL_PATH)
+        return self._model
+
+    def predict(self, frame: np.ndarray) -> dict:
+        return self._ensure_model().predict(frame)
+
+    def reset(self) -> None:
+        if self._model is not None:
+            self._model.reset()
+
+
+_default_model = _OpenWakeWordModel()
+
+
+def listen_for_wake_word(
+    stream,
+    should_continue=lambda: True,
+    model: Optional[WakeModel] = None,
+    threshold: Union[float, Callable[[], float], None] = None,
+    on_tick=None,
+    on_score: Optional[Callable[[float, float], None]] = None,
+) -> bool:
+    """Block until the wake word fires (returns True) or *should_continue*
+    goes false (returns False).
+
+    Feeds every frame to *model* (the thunderbolt openWakeWord model by
+    default) and treats any class score >= *threshold* as a detection,
+    resetting the model's internal state afterward so the next call starts
+    clean — same pattern as desk_client/bolt_desk.py's main loop.
+
+    *threshold* may be a number or a zero-argument callable. The callable
+    form exists because this function blocks for minutes at a time: the
+    tray's wake-word tuner slider has to be able to change sensitivity
+    *during* a listen, not only at the start of the next one.
+
+    *on_tick*, if given, is called once per ``WAKE_CHECK_INTERVAL_SECONDS``
+    (not every frame — prediction is cheap enough to run on every frame, but
+    this is the only point control returns to the caller while otherwise
+    blocked here for a possibly long time, so it's how a caller drives
+    periodic work, e.g. the heartbeat/announcement poll in controller.py,
+    during quiet stretches with no wake word).
+
+    *on_score*, if given, gets ``(best_score, threshold)`` every frame — used
+    to log near misses for threshold tuning.
+    """
+    model = model or _default_model
+    if threshold is None:
+        threshold = config.WAKE_WORD_THRESHOLD
+    resolve_threshold = threshold if callable(threshold) else (lambda: threshold)
+
+    frame_len = config.FRAME_LEN
+    check_every_frames = max(1, int(config.WAKE_CHECK_INTERVAL_SECONDS * config.SAMPLE_RATE / frame_len))
+    frames_since_tick = 0
+
+    while should_continue():
+        chunk, _ = stream.read(frame_len)
+        frame = np.asarray(chunk)[:, 0]
+        scores = model.predict(frame)
+        current_threshold = resolve_threshold()
+        best = max(scores.values()) if scores else 0.0
+
+        frames_since_tick += 1
+        if frames_since_tick >= check_every_frames:
+            frames_since_tick = 0
+            if on_tick is not None:
+                on_tick()
+
+        if on_score is not None:
+            on_score(best, current_threshold)
+
+        if scores and best >= current_threshold:
+            model.reset()
+            return True
+    return False