TheMajesticNetwork/Bolt-Pet
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Update desktop app to android app capabilities.

Browse Source
This commit is contained in:
themajesticmagician
2026-09-13 16:23:52 -06:00
parent 3a0959f55d
commit 2a2cf38399
13 changed files with 485 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bolt_pet/audio/stt.py
+17 -26
View File
@@ -1,14 +1,20 @@
"""Speech-to-text for the actual query, after the wake word fires.
Deepgram, same as desk_client/bolt_desk.py.
The "guaranteed" fallback for when stt_stream's opportunistic live-feed
session didn't produce a transcript (streaming disabled, or the socket
never came up). There is no separate one-shot REST endpoint server-side any
more — /desk/stt is a websocket relay only — so this connects the exact
same way stt_stream.py does and just feeds the whole buffered utterance in
one go instead of frame-by-frame as it's captured. That connection is
unconditional (not gated by STT_STREAMING, which only controls the
opportunistic optimisation), since there is nothing left to fall back to
if it fails.
"""
from __future__ import annotations
import requests
from .. import config
from .mic import pcm_to_wav_bytes
from . import stt_stream
class SttError(Exception):
@@ -16,27 +22,12 @@ class SttError(Exception):
def transcribe(pcm) -> str:
if not config.DEEPGRAM_API_KEY:
raise SttError("DEEPGRAM_API_KEY is not set")
if not config.is_configured():
raise SttError("BOLT_SERVER_URL / DESK_API_KEY not set")
try:
response = requests.post(
"https://api.deepgram.com/v1/listen",
params={"model": config.DEEPGRAM_MODEL, "language": "en", "smart_format": "true"},
headers={
"Authorization": f"Token {config.DEEPGRAM_API_KEY}",
"Content-Type": "audio/wav",
},
data=pcm_to_wav_bytes(pcm),
timeout=30,
)
response.raise_for_status()
socket = stt_stream.connect()
except Exception as exc:
raise SttError(f"transcription request failed: {exc}") from exc
try:
return (
response.json()
.get("results", {}).get("channels", [{}])[0]
.get("alternatives", [{}])[0].get("transcript", "")
).strip()
except Exception as exc:
raise SttError(f"couldn't parse transcription response: {exc}") from exc
raise SttError(f"couldn't reach the transcription server: {exc}") from exc
session = stt_stream.StreamingTranscriber(socket)
session.feed(pcm)
return session.finish()
bolt_pet/audio/stt_stream.py
+46 -25
View File
@@ -1,21 +1,26 @@
"""Streaming speech-to-text — transcribing *while* you talk, not after.
The one-shot path (`stt.transcribe`) waits for the utterance to finish, then
uploads the whole WAV and waits again. That second wait is dead time between
sends the whole clip and waits again. That second wait is dead time between
you stopping and the pet reacting, and it grows with the length of what you
said — a thirty-second question costs noticeably more than a five-second one.
Deepgram's live endpoint removes it: frames go up as they are captured, so by
the time the VAD decides you have stopped, the transcript is essentially
already there. Same model, same account, same accuracy — the difference is
purely when the work happens.
This connects to the Bolt server's own `/desk/stt` — the same websocket relay
the Android app uses — which forwards audio to Deepgram and Deepgram's JSON
messages back untouched. Frames go up as they are captured, so by the time the
VAD decides you have stopped, the transcript is essentially already there.
There is no local Deepgram account or API key any more; auth is this pet's own
`DESK_API_KEY`, same as every other call to the server.
Design constraints that shaped this:
- **Failure must be invisible.** No websocket, no network, a mid-utterance
disconnect — all of it falls back to the one-shot path, which still has the
full audio buffered. Streaming is an optimisation, never a dependency, so
`open()` returning None is an ordinary outcome rather than an error.
full audio buffered. Streaming is an optimisation here, never a dependency —
`available()`/`open()` returning None/False is an ordinary outcome, not an
error. `stt.transcribe()` (the *guaranteed* fallback) talks to the exact same
server relay via `connect()` directly, bypassing that opportunistic gate,
since there is no second, different backend left to fall back to.
- **The VAD still decides when you stopped.** Deepgram has its own endpointing
and using it would save more, but it would also move a decision the rest of
the pipeline is built around (barge-in, follow-up listening, the grace
@@ -42,16 +47,14 @@ from .. import config
logger = logging.getLogger("bolt_pet.stt_stream")
_ENDPOINT = (
"wss://api.deepgram.com/v1/listen"
"?encoding=linear16&channels=1&sample_rate={rate}&model={model}"
"&language=en&smart_format=true&interim_results=false"
)
def available() -> bool:
"""Whether streaming STT can even be attempted in this install."""
if not config.STT_STREAMING or not config.DEEPGRAM_API_KEY:
"""Whether the streaming (transcribe-while-talking) optimization should
be attempted opportunistically. Not a gate on transcription itself — the
server relay is the only way to transcribe at all now, so
stt.transcribe() connects via connect() directly rather than through
this, and isn't affected by STT_STREAMING being off."""
if not config.STT_STREAMING or not config.is_configured():
return False
try:
import websocket # noqa: F401 (websocket-client)
@@ -60,6 +63,33 @@ def available() -> bool:
return False
def _connect(sample_rate: int = None):
"""Open a websocket to the server's `/desk/stt` relay. Raises on any
failure — this is the "no fallback left" connector `stt.transcribe()`
uses directly, as well as the default for `StreamingTranscriber.open()`.
The server chooses the STT model and the endpointing behaviour; this
only states the audio format about to be sent, which is fixed by the
wake model upstream of it."""
import websocket
rate = sample_rate or config.SAMPLE_RATE
url = (
config.SERVER_URL.replace("http", "ws", 1)
+ "/desk/stt"
+ f"?session_id={config.SESSION_ID}&encoding=linear16&sample_rate={rate}"
)
return websocket.create_connection(
url, header=[f"X-Desk-Api-Key: {config.API_KEY}"], timeout=10,
)
# Public name for external callers (stt.py, tests) — named separately from
# the module-private def so StreamingTranscriber.open()'s `connect` parameter
# can shadow the bare name locally without losing access to this.
connect = _connect
class StreamingTranscriber:
"""One utterance's worth of live transcription.
@@ -95,16 +125,7 @@ class StreamingTranscriber:
return None
rate = sample_rate or config.SAMPLE_RATE
try:
if connect is not None:
socket = connect()
else:
import websocket
socket = websocket.create_connection(
_ENDPOINT.format(rate=rate, model=config.DEEPGRAM_MODEL),
header={"Authorization": f"Token {config.DEEPGRAM_API_KEY}"},
timeout=10,
)
socket = connect() if connect is not None else _connect(rate)
return cls(socket, sample_rate=rate)
except Exception as exc:
logger.info("Streaming STT unavailable (%s) — using the one-shot path.", exc)
bolt_pet/audio/tts.py
+33 -37
View File
@@ -1,15 +1,24 @@
"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just
"""Text-to-speech: the Bolt server's own `/desk/tts` — the same endpoint the
Android app streams from — requested as raw PCM so playback is just
sounddevice — no external player binary (mpv/ffplay), unlike
desk_client/bolt_desk.py which shells out because it only targets Linux.
Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
fails, so the pet can still talk with zero cloud config.
No local ElevenLabs account needed for this: the server picks the voice
(ELEVENLABS_VOICE_ID, or a `speak_as` override) and the synthesis model
itself, authenticated with this pet's own DESK_API_KEY. Falls back to
pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech on macOS,
espeak on Linux) if the server call fails, so the pet can still talk even
with the server unreachable.
Every entry point takes an optional *voice_id* that overrides
`ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as`
reply marker reaches the speakers (see controller._apply_voice). The offline
fallback has no such concept and always sounds like itself.
`synthesize_dialogue()` below is the one exception: multi-voice
`dialoguectl` scenes have no server endpoint, so that one call still goes
to ElevenLabs' Text to Dialogue API directly and still needs
ELEVENLABS_API_KEY — see dialogue.py.
"""
from __future__ import annotations
@@ -33,19 +42,8 @@ def voice_for(voice_id: Optional[str] = None) -> str:
return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID
def model_for(text: str, voice_id: Optional[str] = None) -> str:
"""Which ElevenLabs model to synthesize with.
The default (`eleven_flash_v2`) is English-only, and both things that
reach this branch mean the reply probably isn't English: a voice the
server picked mid-conversation is nearly always about a language or an
accent, and non-ASCII text can't be English at all. Rendering either one
through the English model gets you a mangled phonetic reading rather
than a failure, which is worse — so those go through the multilingual
model instead."""
if (voice_id or "").strip() or not text.isascii():
return config.ELEVENLABS_MULTILINGUAL_MODEL_ID
return config.ELEVENLABS_MODEL_ID
def _headers() -> dict:
return {"X-Desk-Api-Key": config.API_KEY}
def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]:
@@ -53,48 +51,46 @@ def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarra
callers should fall back to speak_offline() rather than treating this
as fatal."""
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
if not (config.is_configured() and voice):
raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": model_for(text, voice_id)},
f"{config.SERVER_URL}/desk/tts",
headers=_headers(),
json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
timeout=60,
)
response.raise_for_status()
except Exception as exc:
raise TtsError(f"ElevenLabs request failed: {exc}") from exc
raise TtsError(f"server tts request failed: {exc}") from exc
pcm = np.frombuffer(response.content, dtype=np.int16)
if pcm.size == 0:
raise TtsError("ElevenLabs returned no audio")
raise TtsError("server returned no audio")
return pcm, config.TTS_SAMPLE_RATE
def stream_pcm(
text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None
) -> Iterator[np.ndarray]:
"""Same audio as synthesize_pcm(), but yielded as it arrives from
ElevenLabs' /stream endpoint so playback can start on the first chunk
(~300ms) instead of after the whole clip is synthesized. Raises TtsError
before yielding anything if the request itself fails, so callers can fall
back cleanly; a mid-stream failure just ends the generator."""
"""Same audio as synthesize_pcm(), but yielded as it arrives from the
server so playback can start on the first chunk instead of after the
whole clip is synthesized. Raises TtsError before yielding anything if
the request itself fails, so callers can fall back cleanly; a mid-stream
failure just ends the generator."""
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
if not (config.is_configured() and voice):
raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": model_for(text, voice_id)},
f"{config.SERVER_URL}/desk/tts",
headers=_headers(),
json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
timeout=60,
stream=True,
)
response.raise_for_status()
except Exception as exc:
raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc
raise TtsError(f"server tts stream request failed: {exc}") from exc
return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes))
bolt_pet/config.py
+29 -19
View File
@@ -54,31 +54,41 @@ WAKE_WORD_THRESHOLD = float(os.environ.get("WAKE_WORD_THRESHOLD", "0.5"))
# heartbeat poll in controller.py during quiet stretches with no wake word.
WAKE_CHECK_INTERVAL_SECONDS = float(os.environ.get("WAKE_CHECK_INTERVAL_SECONDS", "1.2"))
# ── STT (Deepgram, same as bolt_desk.py) ────────────────────────────────────
# ── STT (server-hosted, same /desk/stt relay the Android app uses) ─────────
# No local Deepgram account needed any more: audio/stt_stream.py opens a
# websocket to this pet's own BOLT_SERVER_URL/DESK_API_KEY, which the server
# relays to Deepgram and meters it against the same credit ledger as a chat turn.
# There is no separate one-shot REST path server-side, so audio/stt.py's
# "guaranteed" fallback uses this exact same connection too — just fed the
# whole utterance at once instead of frame-by-frame.
DEEPGRAM_API_KEY = os.environ.get("DEEPGRAM_API_KEY", "")
DEEPGRAM_MODEL = os.environ.get("DEEPGRAM_MODEL", "nova-3")
# Transcribe *while* you talk instead of uploading the finished clip: frames go
# up as they are captured, so the transcript is ready the moment the VAD says
# you stopped. Needs `websocket-client`; falls back to the one-shot upload
# whenever it can't connect, so turning it on can only help.
# Transcribe *while* you talk instead of waiting for the utterance to end:
# frames go up as they are captured, so the transcript is ready the moment the
# VAD says you stopped. Needs `websocket-client` (in requirements.txt); when
# it can't connect the one-shot path (audio/stt.py) still tries the same
# server relay itself, so turning this off only costs latency, not the
# ability to transcribe at all.
STT_STREAMING = os.environ.get("STT_STREAMING", "true").lower() in ("1", "true", "yes", "on")
# ── TTS (ElevenLabs, requested as raw PCM so playback needs no external
# player binary — cross-platform via sounddevice instead of shelling out to
# mpv/ffplay like bolt_desk.py does on Linux) ───────────────────────────────
# ── TTS (server-hosted, same /desk/tts endpoint the Android app uses) ──────
# No local ElevenLabs account needed for the normal reply voice any more —
# audio/tts.py posts to this pet's own BOLT_SERVER_URL/DESK_API_KEY and gets
# back raw 16 kHz mono PCM16, same as the phone. Falls back to offline
# pyttsx3 if the server call fails.
#
# ELEVENLABS_API_KEY is still read directly by this client for exactly one
# feature the server has no endpoint for: multi-voice `dialoguectl` scenes
# (audio/tts.synthesize_dialogue, see dialogue.py) — leave it blank and
# everything except that one feature works with zero local API keys.
ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY", "")
# Which voice to ask the server for — an ElevenLabs voice id (or a "vb:"-
# prefixed cloned voice, if this desk key owns one). The server picks the
# synthesis model itself now; there is nothing left for this client to choose.
ELEVENLABS_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID", "")
ELEVENLABS_MODEL_ID = os.environ.get("ELEVENLABS_MODEL_ID", "eleven_flash_v2")
# eleven_flash_v2 is English-only, and the two cases that swap the voice
# (server-picked `speak_as`, or a reply with non-ASCII in it) are usually
# exactly the cases where the reply isn't English — see tts.model_for().
ELEVENLABS_MULTILINGUAL_MODEL_ID = os.environ.get(
"ELEVENLABS_MULTILINGUAL_MODEL_ID", "eleven_flash_v2_5"
)
# ElevenLabs PCM output formats are named pcm_<sample_rate>.
TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "24000"))
# Fixed by the server (ai/desk_media.py's PCM_SAMPLE_RATE) — not a free
# tunable any more, but still an env override in case that ever changes.
TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "16000"))
# Does a voice the server picks (its speak_as marker — "talk like a pirate",
# "say that in Japanese") stay on for later replies, or last one reply only?
bolt_pet/doctor.py
+28 -15
View File
@@ -137,29 +137,42 @@ def check_wake_model() -> Check:
def check_stt() -> Check:
if not config.DEEPGRAM_API_KEY:
return Check("speech-to-text", FAIL, "no DEEPGRAM_API_KEY",
"set it in .env — nothing you say can be transcribed without it")
if config.STT_STREAMING and not _module("websocket"):
return Check("speech-to-text", WARN, "streaming on, but websocket-client is missing",
"pip install websocket-client — it falls back to one-shot uploads")
mode = "streaming" if config.STT_STREAMING else "one-shot"
return Check("speech-to-text", OK, f"Deepgram {config.DEEPGRAM_MODEL}, {mode}")
"""STT is a websocket relay to the server (/desk/stt) — no local Deepgram
account, but websocket-client is now load-bearing for transcription to
work at all, not just the streaming optimisation (there's no separate
REST fallback any more)."""
if not config.is_configured():
return Check("speech-to-text", FAIL, "no BOLT_SERVER_URL/DESK_API_KEY",
"set them in .env — nothing you say can be transcribed without the server")
if not _module("websocket"):
return Check("speech-to-text", FAIL, "websocket-client is missing",
"pip install -r requirements.txt — /desk/stt is a websocket relay "
"with no REST fallback")
mode = "streaming" if config.STT_STREAMING else "one-shot (still via the server relay)"
return Check("speech-to-text", OK, f"server relay, {mode}")
def check_tts() -> Check:
if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
if _module("pyttsx3"):
return Check("text-to-speech", WARN, "no ElevenLabs key/voice — offline voice only",
"set ELEVENLABS_API_KEY and ELEVENLABS_VOICE_ID for the real voice")
return Check("text-to-speech", FAIL, "no ElevenLabs config and no pyttsx3 fallback")
return Check("text-to-speech", OK,
f"ElevenLabs {config.ELEVENLABS_MODEL_ID}, voice …{config.ELEVENLABS_VOICE_ID[-6:]}")
"""TTS is the server's /desk/tts — no local ElevenLabs account needed for
the normal reply voice, just a voice id for it to request."""
if config.is_configured() and config.ELEVENLABS_VOICE_ID:
return Check("text-to-speech", OK,
f"server relay, voice …{config.ELEVENLABS_VOICE_ID[-6:]}")
if _module("pyttsx3"):
return Check("text-to-speech", WARN, "server/voice not configured — offline voice only",
"set BOLT_SERVER_URL/DESK_API_KEY and ELEVENLABS_VOICE_ID for the real voice")
return Check("text-to-speech", FAIL, "no server/voice config and no pyttsx3 fallback",
"set BOLT_SERVER_URL/DESK_API_KEY/ELEVENLABS_VOICE_ID, "
"or pip install pyttsx3 for an offline voice")
def check_dialogue() -> Check:
if not config.DIALOGUE:
return Check("multi-voice scenes", WARN, "disabled (DIALOGUE=false)")
if not config.ELEVENLABS_API_KEY:
return Check("multi-voice scenes", WARN, "no ELEVENLABS_API_KEY",
"set it in .env — dialogue scenes are the one feature still calling "
"ElevenLabs directly, since the server has no equivalent endpoint")
from . import dialogue
cast = dialogue.parse_voice_map(config.DIALOGUE_VOICES)