themajesticmagicianandClaude Fable 5
ca65670638
stt.finish() used to run inline in the frame loop, deafening the assistant for up to ~4s per utterance while the server returned the transcript (and whatever was said meanwhile was later drained). The finish now runs on a background thread per utterance; the segmenter keeps consuming mic frames, so speech during the wait becomes its own utterance and is transcribed too. Transcripts are delivered oldest- first between frames, deferred while an utterance is in progress, and the command/follow-up window no longer times out on transcription latency. Speaking stays half-duplex (echo protection). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
316 lines
13 KiB
Python
316 lines
13 KiB
Python
"""The assistant: hear the wake phrase, take the request, answer out loud.
|
|
|
|
idle every utterance the local detector passes on is transcribed by
|
|
the server; only ones that open with a wake phrase count
|
|
"Hey Bolt, what's the time" -> ask "what's the time"
|
|
"Hey Bolt" -> chime, go to `command`
|
|
command the next thing said (within COMMAND_WAIT_SEC) is the request
|
|
follow_up the reply ended on a question: the next thing said (within
|
|
FOLLOW_UP_SEC) is the answer, no wake phrase needed
|
|
|
|
Speaking is half-duplex: the mic is ignored while Bolt talks and for a
|
|
moment after, so his own voice through the speakers can't wake him (his
|
|
replies say "Bolt" often enough).
|
|
|
|
Transcribing is not: waiting for the server's transcript happens on a
|
|
background thread, so the mic keeps being segmented meanwhile and whatever
|
|
is said next becomes its own utterance and is sent off too. Finished
|
|
transcripts are delivered oldest-first between mic frames.
|
|
|
|
Everything that touches hardware or the network is injected, so the flow
|
|
is tested with fakes; __main__ wires in the real mic, server and speaker.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import platform
|
|
import queue
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from typing import Callable, Optional
|
|
|
|
from . import audio, config, server, speech_text
|
|
from .listener import Segmenter
|
|
from .stt import Transcription
|
|
from .tools import Tools
|
|
from .wake import find_wake
|
|
|
|
ECHO_GUARD_SEC = 0.4 # ignore the mic this long after speaking
|
|
|
|
|
|
def os_label() -> str:
|
|
try:
|
|
return f"{platform.system()} {platform.release()}".strip() or "a computer"
|
|
except Exception:
|
|
return "a computer"
|
|
|
|
|
|
def annotate(text: str, commands: str) -> str:
|
|
"""The note on every request: it's voice, and what Bolt may do here."""
|
|
can = {
|
|
"off": "You can't act on this computer from here.",
|
|
"read": ('You can list and read files with your command tool, e.g. '
|
|
'`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real '
|
|
"output comes back. You can't run shell commands or change files from here."),
|
|
"all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` '
|
|
"for files (list, read, write, edit) or `command: <shell command>`; the real "
|
|
"output comes back."),
|
|
}[commands]
|
|
return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was "
|
|
"spoken and transcribed, so it may contain recognition mistakes. Your reply is read "
|
|
"aloud: keep it short and conversational, with no markdown, lists, links or code. "
|
|
f"{can} Never make up a command's output.]")
|
|
|
|
|
|
class _PendingTranscript:
|
|
"""One utterance already sent off, its transcript still on the way."""
|
|
|
|
__slots__ = ("thread", "text", "error")
|
|
|
|
def __init__(self):
|
|
self.thread: Optional[threading.Thread] = None
|
|
self.text = ""
|
|
self.error: Optional[Exception] = None
|
|
|
|
|
|
class Assistant:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
transcription: Callable = Transcription,
|
|
converse: Callable = server.converse,
|
|
converse_stream: Optional[Callable] = server.converse_stream,
|
|
tts: Callable = server.tts_stream,
|
|
play: Callable = audio.play,
|
|
tools: Optional[Tools] = None,
|
|
log: Callable[[str], None] = print,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
wake_words=None,
|
|
anywhere_words=None,
|
|
verbose: bool = False,
|
|
):
|
|
self._transcription = transcription
|
|
self._converse = converse
|
|
self._stream = converse_stream if config.STREAMING_REPLIES else None
|
|
self._tts, self._play = tts, play
|
|
self.log, self.clock, self.verbose = log, clock, verbose
|
|
self.tools = tools or Tools(log=log)
|
|
self.wake_words = wake_words or config.WAKE_WORDS
|
|
self.anywhere_words = config.ANYWHERE_WAKE_WORDS if anywhere_words is None else anywhere_words
|
|
self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES,
|
|
config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC,
|
|
config.MAX_UTTERANCE_SEC)
|
|
self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000) # ~160 s of audio
|
|
self.state = "idle"
|
|
self._deadline = 0.0
|
|
self._follow_ups = 0
|
|
self._stt = None
|
|
self._pending: "deque[_PendingTranscript]" = deque()
|
|
self._mute_until = 0.0
|
|
self._voice = "" # a voice the server picked (speak_as) sticks for later replies
|
|
self._tts_failed = False
|
|
|
|
# ── audio in ─────────────────────────────────────────────────────────
|
|
|
|
def on_mic_frame(self, frame: bytes) -> None:
|
|
"""Called from the audio thread: queue it, never block."""
|
|
try:
|
|
self.frames.put_nowait(frame)
|
|
except queue.Full:
|
|
pass # we've fallen far behind; dropping audio beats unbounded memory
|
|
|
|
def run(self, stop: threading.Event) -> None:
|
|
self.log(self.listening_banner())
|
|
while not stop.is_set():
|
|
try:
|
|
frame = self.frames.get(timeout=0.2)
|
|
except queue.Empty:
|
|
self._check_deadline()
|
|
else:
|
|
self.process(frame)
|
|
self.handle_transcripts()
|
|
|
|
def listening_banner(self) -> str:
|
|
return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.'
|
|
|
|
def process(self, frame: bytes) -> None:
|
|
if self.clock() < self._mute_until:
|
|
return
|
|
self._check_deadline()
|
|
for event in self.segmenter.feed(frame):
|
|
if event.kind == "start":
|
|
self._stt = self._transcription()
|
|
self._stt.add(event.frames)
|
|
elif event.kind == "frame" and self._stt is not None:
|
|
self._stt.add(event.frames)
|
|
elif event.kind == "dropped":
|
|
if self._stt is not None:
|
|
self._stt.cancel()
|
|
self._stt = None
|
|
elif event.kind == "end":
|
|
stt, self._stt = self._stt, None
|
|
if stt is not None:
|
|
self._start_finish(stt, event.audio)
|
|
|
|
def _start_finish(self, stt, utterance: bytes) -> None:
|
|
"""finish() waits on the server; done here, that wait would deafen
|
|
the loop. Run it on a thread and keep segmenting the mic meanwhile."""
|
|
pending = _PendingTranscript()
|
|
|
|
def work() -> None:
|
|
try:
|
|
pending.text = stt.finish(utterance)
|
|
except Exception as exc: # SttError, or the socket misbehaving
|
|
pending.error = exc
|
|
|
|
pending.thread = threading.Thread(target=work, name="stt-finish", daemon=True)
|
|
self._pending.append(pending)
|
|
pending.thread.start()
|
|
|
|
def handle_transcripts(self, wait: bool = False) -> None:
|
|
"""Deliver finished transcripts, oldest first. One still in flight —
|
|
or an utterance still being spoken — ends the pass rather than
|
|
blocking, unless *wait* (tests, shutdown). The head stays queued
|
|
while heard() runs so _check_deadline knows speech already happened."""
|
|
while self._pending:
|
|
if self.segmenter.in_speech and not wait:
|
|
return
|
|
head = self._pending[0]
|
|
if wait:
|
|
head.thread.join()
|
|
if head.thread.is_alive():
|
|
return
|
|
if head.error is not None:
|
|
self._pending.popleft()
|
|
self.log(f"! transcription failed: {head.error}")
|
|
if self.state != "idle":
|
|
self._chime(audio.ERROR_CHIME)
|
|
continue
|
|
try:
|
|
self.heard(head.text)
|
|
finally:
|
|
self._pending.popleft()
|
|
|
|
def _check_deadline(self) -> None:
|
|
# A pending transcript means something *was* said in time — it's
|
|
# just still at the server; the window mustn't close on that latency.
|
|
if self.state in ("command", "follow_up") and self.clock() > self._deadline \
|
|
and not self.segmenter.in_speech and not self._pending:
|
|
if self.verbose:
|
|
self.log(" (stopped waiting)")
|
|
self.state = "idle"
|
|
|
|
# ── what was said ────────────────────────────────────────────────────
|
|
|
|
def heard(self, text: str) -> None:
|
|
text = (text or "").strip()
|
|
if not text:
|
|
return
|
|
self._check_deadline()
|
|
woke, rest = find_wake(text, self.wake_words, self.anywhere_words)
|
|
if self.state == "idle":
|
|
if not woke:
|
|
if self.verbose:
|
|
self.log(f" (not for me) {text}")
|
|
return
|
|
if not rest:
|
|
self.log("* Yes?")
|
|
self._chime(audio.WAKE_CHIME)
|
|
self._wait("command", config.COMMAND_WAIT_SEC)
|
|
return
|
|
self._follow_ups = 0
|
|
self.ask(rest)
|
|
return
|
|
# command / follow_up: whatever was said is the request. A repeated
|
|
# "Hey Bolt" in front of it is dropped; on its own it just re-arms.
|
|
if woke and not rest:
|
|
self._chime(audio.WAKE_CHIME)
|
|
self._wait(self.state, config.COMMAND_WAIT_SEC)
|
|
return
|
|
if self.state == "command":
|
|
self._follow_ups = 0
|
|
self.ask(rest if woke else text)
|
|
|
|
def _wait(self, state: str, seconds: float) -> None:
|
|
self.state = state
|
|
self._deadline = self.clock() + seconds
|
|
|
|
# ── asking and answering ─────────────────────────────────────────────
|
|
|
|
def ask(self, request: str) -> None:
|
|
self.state = "busy"
|
|
self.log(f"> {request}")
|
|
spoken = []
|
|
|
|
def say(sentence: str) -> None:
|
|
spoken.append(sentence)
|
|
self.speak(sentence)
|
|
|
|
message = annotate(request, self.tools.mode)
|
|
try:
|
|
reply = None
|
|
if self._stream is not None:
|
|
try:
|
|
reply = self._stream(message, on_say=say, on_command=self.tools)
|
|
except server.ServerError:
|
|
reply = None # nothing was said yet: the plain path is invisible
|
|
if reply is None:
|
|
reply = self._converse(message, on_command=self.tools, on_say=say)
|
|
except server.ServerError as exc:
|
|
self.log(f"! {exc}")
|
|
self._chime(audio.ERROR_CHIME)
|
|
self._back_to_listening("idle")
|
|
return
|
|
if reply.voice_id:
|
|
self._voice = reply.voice_id
|
|
if not reply.streamed and reply.text.strip():
|
|
spoken.append(reply.text)
|
|
self.speak(reply.text)
|
|
full = " ".join(spoken)
|
|
if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS:
|
|
self._follow_ups += 1
|
|
self._back_to_listening("follow_up", config.FOLLOW_UP_SEC)
|
|
else:
|
|
self._follow_ups = 0
|
|
self._back_to_listening("idle")
|
|
|
|
def speak(self, text: str) -> None:
|
|
words = speech_text.for_speech(text)
|
|
if not words:
|
|
return
|
|
self.log(f"< {speech_text.for_display(text)}")
|
|
try:
|
|
self._play(self._tts(words, self._voice))
|
|
self._tts_failed = False
|
|
except server.TtsError as exc:
|
|
if not self._tts_failed: # say it once, not once per sentence
|
|
self.log(f"! can't speak: {exc}")
|
|
self._tts_failed = True
|
|
except Exception as exc:
|
|
self.log(f"! playback failed: {exc}")
|
|
|
|
def _chime(self, sound: bytes) -> None:
|
|
if not config.CHIMES:
|
|
return
|
|
try:
|
|
self._play([sound])
|
|
except Exception:
|
|
pass
|
|
self._mute_until = self.clock() + ECHO_GUARD_SEC
|
|
|
|
def _back_to_listening(self, state: str, seconds: float = 0.0) -> None:
|
|
"""Drop what the mic picked up while Bolt was talking (his own voice),
|
|
start detection fresh, and give the room a moment to go quiet."""
|
|
while True:
|
|
try:
|
|
self.frames.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
self.segmenter.reset()
|
|
if self._stt is not None:
|
|
self._stt.cancel()
|
|
self._stt = None
|
|
self._mute_until = self.clock() + ECHO_GUARD_SEC
|
|
self._wait(state, seconds + ECHO_GUARD_SEC)
|