Upload

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

bolt_pet/audio/tts.py

@@ -0,0 +1,163 @@
+"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just
+sounddevice — no external player binary (mpv/ffplay), unlike
+desk_client/bolt_desk.py which shells out because it only targets Linux.
+
+Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
+on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
+fails, so the pet can still talk with zero cloud config.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Iterator
+
+import numpy as np
+import requests
+
+from .. import config, speech_text
+
+
+class TtsError(Exception):
+    pass
+
+
+def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
+    """Returns (pcm_int16_mono, sample_rate). Raises TtsError on failure —
+    callers should fall back to speak_offline() rather than treating this
+    as fatal."""
+    if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
+        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+    try:
+        response = requests.post(
+            f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}",
+            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
+            timeout=60,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise TtsError(f"ElevenLabs request failed: {exc}") from exc
+    pcm = np.frombuffer(response.content, dtype=np.int16)
+    if pcm.size == 0:
+        raise TtsError("ElevenLabs returned no audio")
+    return pcm, config.TTS_SAMPLE_RATE
+
+
+def stream_pcm(text: str, chunk_bytes: int = 4096) -> Iterator[np.ndarray]:
+    """Same audio as synthesize_pcm(), but yielded as it arrives from
+    ElevenLabs' /stream endpoint so playback can start on the first chunk
+    (~300ms) instead of after the whole clip is synthesized. Raises TtsError
+    before yielding anything if the request itself fails, so callers can fall
+    back cleanly; a mid-stream failure just ends the generator."""
+    if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
+        raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
+    try:
+        response = requests.post(
+            f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}/stream",
+            headers={"xi-api-key": config.ELEVENLABS_API_KEY},
+            params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
+            json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
+            timeout=60,
+            stream=True,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc
+    return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes))
+
+
+def chunks_to_int16(byte_chunks: Iterable[bytes]) -> Iterator[np.ndarray]:
+    """Reassemble a byte stream into int16 frames. HTTP chunk boundaries fall
+    wherever they like, including *inside* a 16-bit sample, so a trailing odd
+    byte has to be carried into the next chunk — otherwise every chunk after
+    the first is shifted by one byte and plays as static."""
+    carry = b""
+    for chunk in byte_chunks:
+        if not chunk:
+            continue
+        data = carry + chunk
+        usable = len(data) - (len(data) % 2)
+        carry = data[usable:]
+        if usable:
+            yield np.frombuffer(data[:usable], dtype=np.int16)
+
+
+def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool:
+    """Play a whole clip. Returns True if it finished, False if *should_stop*
+    (barge-in) cut it short. *should_stop* is polled while audio plays — each
+    poll consumes one mic frame, which is what paces this loop."""
+    import sounddevice as sd
+
+    sd.play(pcm, samplerate=sample_rate, device=config.SPEAKER_DEVICE)
+    if not blocking:
+        return True
+    if should_stop is None:
+        sd.wait()
+        return True
+    while True:
+        try:
+            if not sd.get_stream().active:
+                break
+        except Exception:
+            break  # stream already torn down — playback is over
+        if should_stop():
+            sd.stop()
+            return False
+    return True
+
+
+def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None) -> bool:
+    """Play int16 chunks as they arrive. Returns False if interrupted."""
+    import sounddevice as sd
+
+    with sd.OutputStream(
+        samplerate=sample_rate, channels=1, dtype="int16", device=config.SPEAKER_DEVICE
+    ) as out:
+        for chunk in chunks:
+            if should_stop is not None and should_stop():
+                # abort() rather than draining: barge-in should stop the voice
+                # now, not at the end of the buffered chunk.
+                out.abort()
+                return False
+            out.write(chunk)
+    return True
+
+
+def speak_offline(text: str) -> None:
+    try:
+        import pyttsx3
+    except ImportError:
+        return  # no TTS available at all — caller already logs the text
+    engine = pyttsx3.init()
+    engine.say(text)
+    engine.runAndWait()
+
+
+def speak(text: str, on_error=None, should_stop=None) -> bool:
+    """Speak *text*, preferring streaming ElevenLabs, then whole-clip
+    ElevenLabs, then offline TTS. *on_error*, if given, is called with the
+    exception when ElevenLabs fails (useful for logging) — a fallback still
+    runs either way. Returns False if barge-in interrupted playback.
+
+    The text is sanitized first (speech_text.for_speech): server replies are
+    written for a chat window, and a voice reads markdown/emoji literally
+    ("asterisk asterisk"). Sanitizing here rather than at the call sites means
+    every path to the speakers — reply, heartbeat announcement — is covered."""
+    text = speech_text.for_speech(text)
+    if not text:
+        return True
+    if config.TTS_STREAMING:
+        try:
+            return play_stream(stream_pcm(text), config.TTS_SAMPLE_RATE, should_stop=should_stop)
+        except TtsError as exc:
+            if on_error is not None:
+                on_error(exc)
+    try:
+        pcm, sample_rate = synthesize_pcm(text)
+        return play_pcm(pcm, sample_rate, should_stop=should_stop)
+    except TtsError as exc:
+        if on_error is not None:
+            on_error(exc)
+    speak_offline(text)
+    return True