TheMajesticNetwork/Bolt-Pet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Add text-to-dialogue, self-restart capability, and misc updates

Browse Source
This commit is contained in:
Aaron Franks
2026-07-30 20:50:41 -06:00
parent 5b49670983
commit 96afc351ac
21 changed files with 1819 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bolt_pet/audio/tts.py
+96 -12
View File
@@ -5,11 +5,16 @@ desk_client/bolt_desk.py which shells out because it only targets Linux.
Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
fails, so the pet can still talk with zero cloud config.
Every entry point takes an optional *voice_id* that overrides
`ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as`
reply marker reaches the speakers (see controller._apply_voice). The offline
fallback has no such concept and always sounds like itself.
"""
from __future__ import annotations
from typing import Iterable, Iterator
from typing import Iterable, Iterator, Optional
import numpy as np
import requests
@@ -21,18 +26,40 @@ class TtsError(Exception):
pass
def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
def voice_for(voice_id: Optional[str] = None) -> str:
"""The voice this call should use: an override (server `speak_as`) if
given, else the configured default."""
return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID
def model_for(text: str, voice_id: Optional[str] = None) -> str:
"""Which ElevenLabs model to synthesize with.
The default (`eleven_flash_v2`) is English-only, and both things that
reach this branch mean the reply probably isn't English: a voice the
server picked mid-conversation is nearly always about a language or an
accent, and non-ASCII text can't be English at all. Rendering either one
through the English model gets you a mangled phonetic reading rather
than a failure, which is worse — so those go through the multilingual
model instead."""
if (voice_id or "").strip() or not text.isascii():
return config.ELEVENLABS_MULTILINGUAL_MODEL_ID
return config.ELEVENLABS_MODEL_ID
def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]:
"""Returns (pcm_int16_mono, sample_rate). Raises TtsError on failure —
callers should fall back to speak_offline() rather than treating this
as fatal."""
if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}",
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
json={"text": text, "model_id": model_for(text, voice_id)},
timeout=60,
)
response.raise_for_status()
@@ -44,20 +71,23 @@ def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
return pcm, config.TTS_SAMPLE_RATE
def stream_pcm(text: str, chunk_bytes: int = 4096) -> Iterator[np.ndarray]:
def stream_pcm(
text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None
) -> Iterator[np.ndarray]:
"""Same audio as synthesize_pcm(), but yielded as it arrives from
ElevenLabs' /stream endpoint so playback can start on the first chunk
(~300ms) instead of after the whole clip is synthesized. Raises TtsError
before yielding anything if the request itself fails, so callers can fall
back cleanly; a mid-stream failure just ends the generator."""
if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}/stream",
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
json={"text": text, "model_id": model_for(text, voice_id)},
timeout=60,
stream=True,
)
@@ -83,6 +113,55 @@ def chunks_to_int16(byte_chunks: Iterable[bytes]) -> Iterator[np.ndarray]:
yield np.frombuffer(data[:usable], dtype=np.int16)
def synthesize_dialogue(
inputs: list, model_id: Optional[str] = None, stability: Optional[float] = None
) -> tuple[np.ndarray, int]:
"""Multi-voice scene via ElevenLabs Text to Dialogue.
One request, one take: the whole exchange is synthesized together, which
is the point — the model hears the previous line, so reactions and timing
land instead of sounding like separately-rendered clips.
Same PCM-over-`requests` posture as the rest of this module (no SDK, no
`play()` shelling out to ffplay), so playback is the same sounddevice path
everything else uses and barge-in works on it unchanged. There is no
documented streaming variant, and a scene is a short set piece anyway, so
this is whole-clip only.
"""
if not (config.ELEVENLABS_API_KEY and inputs):
raise TtsError("ELEVENLABS_API_KEY not set (or no dialogue lines)")
body: dict = {
"inputs": [
{"text": str(entry.get("text") or ""), "voice_id": str(entry.get("voice_id") or "")}
for entry in inputs
],
"model_id": model_id or config.DIALOGUE_MODEL_ID,
}
if stability is not None:
body["settings"] = {"stability": float(stability)}
try:
response = requests.post(
"https://api.elevenlabs.io/v1/text-to-dialogue",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json=body,
timeout=120, # a multi-voice take is slower to render than one line
)
response.raise_for_status()
except Exception as exc:
detail = ""
# The API explains refusals (character limit, unknown voice) in the
# body; surfacing it is what lets Bolt fix the call and retry.
body_text = getattr(getattr(exc, "response", None), "text", "")
if body_text:
detail = f" — {body_text[:300]}"
raise TtsError(f"ElevenLabs dialogue request failed: {exc}{detail}") from exc
pcm = np.frombuffer(response.content, dtype=np.int16)
if pcm.size == 0:
raise TtsError("ElevenLabs returned no dialogue audio")
return pcm, config.TTS_SAMPLE_RATE
def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool:
"""Play a whole clip. Returns True if it finished, False if *should_stop*
(barge-in) cut it short. *should_stop* is polled while audio plays — each
@@ -134,11 +213,12 @@ def speak_offline(text: str) -> None:
engine.runAndWait()
def speak(text: str, on_error=None, should_stop=None) -> bool:
def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None) -> bool:
"""Speak *text*, preferring streaming ElevenLabs, then whole-clip
ElevenLabs, then offline TTS. *on_error*, if given, is called with the
exception when ElevenLabs fails (useful for logging) — a fallback still
runs either way. Returns False if barge-in interrupted playback.
*voice_id* overrides the configured voice for this line only.
The text is sanitized first (speech_text.for_speech): server replies are
written for a chat window, and a voice reads markdown/emoji literally
@@ -149,12 +229,16 @@ def speak(text: str, on_error=None, should_stop=None) -> bool:
return True
if config.TTS_STREAMING:
try:
return play_stream(stream_pcm(text), config.TTS_SAMPLE_RATE, should_stop=should_stop)
return play_stream(
stream_pcm(text, voice_id=voice_id),
config.TTS_SAMPLE_RATE,
should_stop=should_stop,
)
except TtsError as exc:
if on_error is not None:
on_error(exc)
try:
pcm, sample_rate = synthesize_pcm(text)
pcm, sample_rate = synthesize_pcm(text, voice_id=voice_id)
return play_pcm(pcm, sample_rate, should_stop=should_stop)
except TtsError as exc:
if on_error is not None: