"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just sounddevice — no external player binary (mpv/ffplay), unlike desk_client/bolt_desk.py which shells out because it only targets Linux. Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech on macOS, espeak on Linux) if ElevenLabs isn't configured or the request fails, so the pet can still talk with zero cloud config. Every entry point takes an optional *voice_id* that overrides `ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as` reply marker reaches the speakers (see controller._apply_voice). The offline fallback has no such concept and always sounds like itself. """ from __future__ import annotations import time from typing import Iterable, Iterator, Optional import numpy as np import requests from .. import config, speech_text class TtsError(Exception): pass def voice_for(voice_id: Optional[str] = None) -> str: """The voice this call should use: an override (server `speak_as`) if given, else the configured default.""" return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID def model_for(text: str, voice_id: Optional[str] = None) -> str: """Which ElevenLabs model to synthesize with. The default (`eleven_flash_v2`) is English-only, and both things that reach this branch mean the reply probably isn't English: a voice the server picked mid-conversation is nearly always about a language or an accent, and non-ASCII text can't be English at all. Rendering either one through the English model gets you a mangled phonetic reading rather than a failure, which is worse — so those go through the multilingual model instead.""" if (voice_id or "").strip() or not text.isascii(): return config.ELEVENLABS_MULTILINGUAL_MODEL_ID return config.ELEVENLABS_MODEL_ID def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]: """Returns (pcm_int16_mono, sample_rate). Raises TtsError on failure — callers should fall back to speak_offline() rather than treating this as fatal.""" voice = voice_for(voice_id) if not (config.ELEVENLABS_API_KEY and voice): raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set") try: response = requests.post( f"https://api.elevenlabs.io/v1/text-to-speech/{voice}", headers={"xi-api-key": config.ELEVENLABS_API_KEY}, params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"}, json={"text": text, "model_id": model_for(text, voice_id)}, timeout=60, ) response.raise_for_status() except Exception as exc: raise TtsError(f"ElevenLabs request failed: {exc}") from exc pcm = np.frombuffer(response.content, dtype=np.int16) if pcm.size == 0: raise TtsError("ElevenLabs returned no audio") return pcm, config.TTS_SAMPLE_RATE def stream_pcm( text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None ) -> Iterator[np.ndarray]: """Same audio as synthesize_pcm(), but yielded as it arrives from ElevenLabs' /stream endpoint so playback can start on the first chunk (~300ms) instead of after the whole clip is synthesized. Raises TtsError before yielding anything if the request itself fails, so callers can fall back cleanly; a mid-stream failure just ends the generator.""" voice = voice_for(voice_id) if not (config.ELEVENLABS_API_KEY and voice): raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set") try: response = requests.post( f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream", headers={"xi-api-key": config.ELEVENLABS_API_KEY}, params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"}, json={"text": text, "model_id": model_for(text, voice_id)}, timeout=60, stream=True, ) response.raise_for_status() except Exception as exc: raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes)) def chunks_to_int16(byte_chunks: Iterable[bytes]) -> Iterator[np.ndarray]: """Reassemble a byte stream into int16 frames. HTTP chunk boundaries fall wherever they like, including *inside* a 16-bit sample, so a trailing odd byte has to be carried into the next chunk — otherwise every chunk after the first is shifted by one byte and plays as static.""" carry = b"" for chunk in byte_chunks: if not chunk: continue data = carry + chunk usable = len(data) - (len(data) % 2) carry = data[usable:] if usable: yield np.frombuffer(data[:usable], dtype=np.int16) def synthesize_dialogue( inputs: list, model_id: Optional[str] = None, stability: Optional[float] = None ) -> tuple[np.ndarray, int]: """Multi-voice scene via ElevenLabs Text to Dialogue. One request, one take: the whole exchange is synthesized together, which is the point — the model hears the previous line, so reactions and timing land instead of sounding like separately-rendered clips. Same PCM-over-`requests` posture as the rest of this module (no SDK, no `play()` shelling out to ffplay), so playback is the same sounddevice path everything else uses and barge-in works on it unchanged. There is no documented streaming variant, and a scene is a short set piece anyway, so this is whole-clip only. """ if not (config.ELEVENLABS_API_KEY and inputs): raise TtsError("ELEVENLABS_API_KEY not set (or no dialogue lines)") body: dict = { "inputs": [ {"text": str(entry.get("text") or ""), "voice_id": str(entry.get("voice_id") or "")} for entry in inputs ], "model_id": model_id or config.DIALOGUE_MODEL_ID, } if stability is not None: body["settings"] = {"stability": float(stability)} try: response = requests.post( "https://api.elevenlabs.io/v1/text-to-dialogue", headers={"xi-api-key": config.ELEVENLABS_API_KEY}, params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"}, json=body, timeout=120, # a multi-voice take is slower to render than one line ) response.raise_for_status() except Exception as exc: detail = "" # The API explains refusals (character limit, unknown voice) in the # body; surfacing it is what lets Bolt fix the call and retry. body_text = getattr(getattr(exc, "response", None), "text", "") if body_text: detail = f" — {body_text[:300]}" raise TtsError(f"ElevenLabs dialogue request failed: {exc}{detail}") from exc pcm = np.frombuffer(response.content, dtype=np.int16) if pcm.size == 0: raise TtsError("ElevenLabs returned no dialogue audio") return pcm, config.TTS_SAMPLE_RATE # ── how loud is it right now ──────────────────────────────────────────────── # The PCM is already decoded here on its way to the speakers, so the amplitude # envelope is free — and it is exactly what a mouth needs to move in time with # speech. Throwing it away and animating the mouth on a timer instead is why # most talking sprites look dubbed. # int16 RMS that counts as "mouth fully open". Speech peaks around 8-12k; # 6000 keeps normal talking in the upper half of the range without clipping # every syllable to wide-open. _LOUD_RMS = 6000.0 def level_of(frame: np.ndarray) -> float: """0..1 loudness for one chunk of PCM. Square-rooted because perceived loudness is not linear in amplitude — a linear mapping leaves the mouth barely moving through ordinary speech.""" if frame is None or len(frame) == 0: return 0.0 rms = float(np.sqrt(np.mean(np.square(frame.astype(np.float32))))) return float(min(1.0, (rms / _LOUD_RMS) ** 0.5)) def envelope(pcm: np.ndarray, sample_rate: int, fps: int = 30) -> list: """Per-frame loudness for a whole clip, for playback that isn't streamed.""" if pcm is None or len(pcm) == 0: return [] window = max(1, int(sample_rate / max(1, fps))) return [level_of(pcm[start:start + window]) for start in range(0, len(pcm), window)] def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None, on_level=None) -> bool: """Play a whole clip. Returns True if it finished, False if *should_stop* (barge-in) cut it short. *should_stop* is polled while audio plays — each poll consumes one mic frame, which is what paces this loop.""" import sounddevice as sd levels = envelope(pcm, sample_rate) if on_level is not None else [] started = time.monotonic() sd.play(pcm, samplerate=sample_rate, device=config.SPEAKER_DEVICE) if not blocking: return True if should_stop is None and on_level is None: sd.wait() return True while True: if levels: # Indexed by elapsed time rather than by chunk, because this path # hands the whole clip to the device at once and never sees it # again — wall clock is the only position we have. index = int((time.monotonic() - started) * 30) if index < len(levels): try: on_level(levels[index]) except Exception: levels = [] try: if not sd.get_stream().active: break except Exception: break # stream already torn down — playback is over if should_stop is not None and should_stop(): sd.stop() return False return True def play_stream(chunks: Iterable[np.ndarray], sample_rate: int, should_stop=None, on_level=None) -> bool: """Play int16 chunks as they arrive. Returns False if interrupted. *on_level* receives each chunk's loudness (0..1) just before it is written, which is what drives the mouth: the sprite is animated by the same audio the speakers are getting, not by a guess about how long a word takes.""" import sounddevice as sd with sd.OutputStream( samplerate=sample_rate, channels=1, dtype="int16", device=config.SPEAKER_DEVICE ) as out: for chunk in chunks: if should_stop is not None and should_stop(): # abort() rather than draining: barge-in should stop the voice # now, not at the end of the buffered chunk. out.abort() return False if on_level is not None: try: on_level(level_of(chunk)) except Exception: on_level = None # a broken listener must not stop playback out.write(chunk) return True def speak_offline(text: str) -> None: try: import pyttsx3 except ImportError: return # no TTS available at all — caller already logs the text engine = pyttsx3.init() engine.say(text) engine.runAndWait() def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None, on_level=None) -> bool: """Speak *text*, preferring streaming ElevenLabs, then whole-clip ElevenLabs, then offline TTS. *on_error*, if given, is called with the exception when ElevenLabs fails (useful for logging) — a fallback still runs either way. Returns False if barge-in interrupted playback. *voice_id* overrides the configured voice for this line only. The text is sanitized first (speech_text.for_speech): server replies are written for a chat window, and a voice reads markdown/emoji literally ("asterisk asterisk"). Sanitizing here rather than at the call sites means every path to the speakers — reply, heartbeat announcement — is covered.""" text = speech_text.for_speech(text) if not text: return True if config.TTS_STREAMING: try: return play_stream( stream_pcm(text, voice_id=voice_id), config.TTS_SAMPLE_RATE, should_stop=should_stop, on_level=on_level, ) except TtsError as exc: if on_error is not None: on_error(exc) try: pcm, sample_rate = synthesize_pcm(text, voice_id=voice_id) return play_pcm(pcm, sample_rate, should_stop=should_stop, on_level=on_level) except TtsError as exc: if on_error is not None: on_error(exc) speak_offline(text) return True