Update desktop app to android app capabilities.
This commit is contained in:
+17
-26
@@ -1,14 +1,20 @@
|
||||
"""Speech-to-text for the actual query, after the wake word fires.
|
||||
|
||||
Deepgram, same as desk_client/bolt_desk.py.
|
||||
The "guaranteed" fallback for when stt_stream's opportunistic live-feed
|
||||
session didn't produce a transcript (streaming disabled, or the socket
|
||||
never came up). There is no separate one-shot REST endpoint server-side any
|
||||
more — /desk/stt is a websocket relay only — so this connects the exact
|
||||
same way stt_stream.py does and just feeds the whole buffered utterance in
|
||||
one go instead of frame-by-frame as it's captured. That connection is
|
||||
unconditional (not gated by STT_STREAMING, which only controls the
|
||||
opportunistic optimisation), since there is nothing left to fall back to
|
||||
if it fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import requests
|
||||
|
||||
from .. import config
|
||||
from .mic import pcm_to_wav_bytes
|
||||
from . import stt_stream
|
||||
|
||||
|
||||
class SttError(Exception):
|
||||
@@ -16,27 +22,12 @@ class SttError(Exception):
|
||||
|
||||
|
||||
def transcribe(pcm) -> str:
|
||||
if not config.DEEPGRAM_API_KEY:
|
||||
raise SttError("DEEPGRAM_API_KEY is not set")
|
||||
if not config.is_configured():
|
||||
raise SttError("BOLT_SERVER_URL / DESK_API_KEY not set")
|
||||
try:
|
||||
response = requests.post(
|
||||
"https://api.deepgram.com/v1/listen",
|
||||
params={"model": config.DEEPGRAM_MODEL, "language": "en", "smart_format": "true"},
|
||||
headers={
|
||||
"Authorization": f"Token {config.DEEPGRAM_API_KEY}",
|
||||
"Content-Type": "audio/wav",
|
||||
},
|
||||
data=pcm_to_wav_bytes(pcm),
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
socket = stt_stream.connect()
|
||||
except Exception as exc:
|
||||
raise SttError(f"transcription request failed: {exc}") from exc
|
||||
try:
|
||||
return (
|
||||
response.json()
|
||||
.get("results", {}).get("channels", [{}])[0]
|
||||
.get("alternatives", [{}])[0].get("transcript", "")
|
||||
).strip()
|
||||
except Exception as exc:
|
||||
raise SttError(f"couldn't parse transcription response: {exc}") from exc
|
||||
raise SttError(f"couldn't reach the transcription server: {exc}") from exc
|
||||
session = stt_stream.StreamingTranscriber(socket)
|
||||
session.feed(pcm)
|
||||
return session.finish()
|
||||
|
||||
@@ -1,21 +1,26 @@
|
||||
"""Streaming speech-to-text — transcribing *while* you talk, not after.
|
||||
|
||||
The one-shot path (`stt.transcribe`) waits for the utterance to finish, then
|
||||
uploads the whole WAV and waits again. That second wait is dead time between
|
||||
sends the whole clip and waits again. That second wait is dead time between
|
||||
you stopping and the pet reacting, and it grows with the length of what you
|
||||
said — a thirty-second question costs noticeably more than a five-second one.
|
||||
|
||||
Deepgram's live endpoint removes it: frames go up as they are captured, so by
|
||||
the time the VAD decides you have stopped, the transcript is essentially
|
||||
already there. Same model, same account, same accuracy — the difference is
|
||||
purely when the work happens.
|
||||
This connects to the Bolt server's own `/desk/stt` — the same websocket relay
|
||||
the Android app uses — which forwards audio to Deepgram and Deepgram's JSON
|
||||
messages back untouched. Frames go up as they are captured, so by the time the
|
||||
VAD decides you have stopped, the transcript is essentially already there.
|
||||
There is no local Deepgram account or API key any more; auth is this pet's own
|
||||
`DESK_API_KEY`, same as every other call to the server.
|
||||
|
||||
Design constraints that shaped this:
|
||||
|
||||
- **Failure must be invisible.** No websocket, no network, a mid-utterance
|
||||
disconnect — all of it falls back to the one-shot path, which still has the
|
||||
full audio buffered. Streaming is an optimisation, never a dependency, so
|
||||
`open()` returning None is an ordinary outcome rather than an error.
|
||||
full audio buffered. Streaming is an optimisation here, never a dependency —
|
||||
`available()`/`open()` returning None/False is an ordinary outcome, not an
|
||||
error. `stt.transcribe()` (the *guaranteed* fallback) talks to the exact same
|
||||
server relay via `connect()` directly, bypassing that opportunistic gate,
|
||||
since there is no second, different backend left to fall back to.
|
||||
- **The VAD still decides when you stopped.** Deepgram has its own endpointing
|
||||
and using it would save more, but it would also move a decision the rest of
|
||||
the pipeline is built around (barge-in, follow-up listening, the grace
|
||||
@@ -42,16 +47,14 @@ from .. import config
|
||||
|
||||
logger = logging.getLogger("bolt_pet.stt_stream")
|
||||
|
||||
_ENDPOINT = (
|
||||
"wss://api.deepgram.com/v1/listen"
|
||||
"?encoding=linear16&channels=1&sample_rate={rate}&model={model}"
|
||||
"&language=en&smart_format=true&interim_results=false"
|
||||
)
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Whether streaming STT can even be attempted in this install."""
|
||||
if not config.STT_STREAMING or not config.DEEPGRAM_API_KEY:
|
||||
"""Whether the streaming (transcribe-while-talking) optimization should
|
||||
be attempted opportunistically. Not a gate on transcription itself — the
|
||||
server relay is the only way to transcribe at all now, so
|
||||
stt.transcribe() connects via connect() directly rather than through
|
||||
this, and isn't affected by STT_STREAMING being off."""
|
||||
if not config.STT_STREAMING or not config.is_configured():
|
||||
return False
|
||||
try:
|
||||
import websocket # noqa: F401 (websocket-client)
|
||||
@@ -60,6 +63,33 @@ def available() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _connect(sample_rate: int = None):
|
||||
"""Open a websocket to the server's `/desk/stt` relay. Raises on any
|
||||
failure — this is the "no fallback left" connector `stt.transcribe()`
|
||||
uses directly, as well as the default for `StreamingTranscriber.open()`.
|
||||
|
||||
The server chooses the STT model and the endpointing behaviour; this
|
||||
only states the audio format about to be sent, which is fixed by the
|
||||
wake model upstream of it."""
|
||||
import websocket
|
||||
|
||||
rate = sample_rate or config.SAMPLE_RATE
|
||||
url = (
|
||||
config.SERVER_URL.replace("http", "ws", 1)
|
||||
+ "/desk/stt"
|
||||
+ f"?session_id={config.SESSION_ID}&encoding=linear16&sample_rate={rate}"
|
||||
)
|
||||
return websocket.create_connection(
|
||||
url, header=[f"X-Desk-Api-Key: {config.API_KEY}"], timeout=10,
|
||||
)
|
||||
|
||||
|
||||
# Public name for external callers (stt.py, tests) — named separately from
|
||||
# the module-private def so StreamingTranscriber.open()'s `connect` parameter
|
||||
# can shadow the bare name locally without losing access to this.
|
||||
connect = _connect
|
||||
|
||||
|
||||
class StreamingTranscriber:
|
||||
"""One utterance's worth of live transcription.
|
||||
|
||||
@@ -95,16 +125,7 @@ class StreamingTranscriber:
|
||||
return None
|
||||
rate = sample_rate or config.SAMPLE_RATE
|
||||
try:
|
||||
if connect is not None:
|
||||
socket = connect()
|
||||
else:
|
||||
import websocket
|
||||
|
||||
socket = websocket.create_connection(
|
||||
_ENDPOINT.format(rate=rate, model=config.DEEPGRAM_MODEL),
|
||||
header={"Authorization": f"Token {config.DEEPGRAM_API_KEY}"},
|
||||
timeout=10,
|
||||
)
|
||||
socket = connect() if connect is not None else _connect(rate)
|
||||
return cls(socket, sample_rate=rate)
|
||||
except Exception as exc:
|
||||
logger.info("Streaming STT unavailable (%s) — using the one-shot path.", exc)
|
||||
|
||||
+33
-37
@@ -1,15 +1,24 @@
|
||||
"""Text-to-speech: ElevenLabs, requested as raw PCM so playback is just
|
||||
"""Text-to-speech: the Bolt server's own `/desk/tts` — the same endpoint the
|
||||
Android app streams from — requested as raw PCM so playback is just
|
||||
sounddevice — no external player binary (mpv/ffplay), unlike
|
||||
desk_client/bolt_desk.py which shells out because it only targets Linux.
|
||||
|
||||
Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
|
||||
on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
|
||||
fails, so the pet can still talk with zero cloud config.
|
||||
No local ElevenLabs account needed for this: the server picks the voice
|
||||
(ELEVENLABS_VOICE_ID, or a `speak_as` override) and the synthesis model
|
||||
itself, authenticated with this pet's own DESK_API_KEY. Falls back to
|
||||
pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech on macOS,
|
||||
espeak on Linux) if the server call fails, so the pet can still talk even
|
||||
with the server unreachable.
|
||||
|
||||
Every entry point takes an optional *voice_id* that overrides
|
||||
`ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as`
|
||||
reply marker reaches the speakers (see controller._apply_voice). The offline
|
||||
fallback has no such concept and always sounds like itself.
|
||||
|
||||
`synthesize_dialogue()` below is the one exception: multi-voice
|
||||
`dialoguectl` scenes have no server endpoint, so that one call still goes
|
||||
to ElevenLabs' Text to Dialogue API directly and still needs
|
||||
ELEVENLABS_API_KEY — see dialogue.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -33,19 +42,8 @@ def voice_for(voice_id: Optional[str] = None) -> str:
|
||||
return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID
|
||||
|
||||
|
||||
def model_for(text: str, voice_id: Optional[str] = None) -> str:
|
||||
"""Which ElevenLabs model to synthesize with.
|
||||
|
||||
The default (`eleven_flash_v2`) is English-only, and both things that
|
||||
reach this branch mean the reply probably isn't English: a voice the
|
||||
server picked mid-conversation is nearly always about a language or an
|
||||
accent, and non-ASCII text can't be English at all. Rendering either one
|
||||
through the English model gets you a mangled phonetic reading rather
|
||||
than a failure, which is worse — so those go through the multilingual
|
||||
model instead."""
|
||||
if (voice_id or "").strip() or not text.isascii():
|
||||
return config.ELEVENLABS_MULTILINGUAL_MODEL_ID
|
||||
return config.ELEVENLABS_MODEL_ID
|
||||
def _headers() -> dict:
|
||||
return {"X-Desk-Api-Key": config.API_KEY}
|
||||
|
||||
|
||||
def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]:
|
||||
@@ -53,48 +51,46 @@ def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarra
|
||||
callers should fall back to speak_offline() rather than treating this
|
||||
as fatal."""
|
||||
voice = voice_for(voice_id)
|
||||
if not (config.ELEVENLABS_API_KEY and voice):
|
||||
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
|
||||
if not (config.is_configured() and voice):
|
||||
raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
|
||||
try:
|
||||
response = requests.post(
|
||||
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}",
|
||||
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
|
||||
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
|
||||
json={"text": text, "model_id": model_for(text, voice_id)},
|
||||
f"{config.SERVER_URL}/desk/tts",
|
||||
headers=_headers(),
|
||||
json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
|
||||
timeout=60,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise TtsError(f"ElevenLabs request failed: {exc}") from exc
|
||||
raise TtsError(f"server tts request failed: {exc}") from exc
|
||||
pcm = np.frombuffer(response.content, dtype=np.int16)
|
||||
if pcm.size == 0:
|
||||
raise TtsError("ElevenLabs returned no audio")
|
||||
raise TtsError("server returned no audio")
|
||||
return pcm, config.TTS_SAMPLE_RATE
|
||||
|
||||
|
||||
def stream_pcm(
|
||||
text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None
|
||||
) -> Iterator[np.ndarray]:
|
||||
"""Same audio as synthesize_pcm(), but yielded as it arrives from
|
||||
ElevenLabs' /stream endpoint so playback can start on the first chunk
|
||||
(~300ms) instead of after the whole clip is synthesized. Raises TtsError
|
||||
before yielding anything if the request itself fails, so callers can fall
|
||||
back cleanly; a mid-stream failure just ends the generator."""
|
||||
"""Same audio as synthesize_pcm(), but yielded as it arrives from the
|
||||
server so playback can start on the first chunk instead of after the
|
||||
whole clip is synthesized. Raises TtsError before yielding anything if
|
||||
the request itself fails, so callers can fall back cleanly; a mid-stream
|
||||
failure just ends the generator."""
|
||||
voice = voice_for(voice_id)
|
||||
if not (config.ELEVENLABS_API_KEY and voice):
|
||||
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
|
||||
if not (config.is_configured() and voice):
|
||||
raise TtsError("BOLT_SERVER_URL / DESK_API_KEY / ELEVENLABS_VOICE_ID not set")
|
||||
try:
|
||||
response = requests.post(
|
||||
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream",
|
||||
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
|
||||
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
|
||||
json={"text": text, "model_id": model_for(text, voice_id)},
|
||||
f"{config.SERVER_URL}/desk/tts",
|
||||
headers=_headers(),
|
||||
json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
|
||||
timeout=60,
|
||||
stream=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise TtsError(f"ElevenLabs stream request failed: {exc}") from exc
|
||||
raise TtsError(f"server tts stream request failed: {exc}") from exc
|
||||
return chunks_to_int16(response.iter_content(chunk_size=chunk_bytes))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user