"""The assistant: hear a request, answer out loud. idle every utterance the local detector passes on is transcribed by the server and, by default, is itself the request — the mic being open is the activation. With REQUIRE_WAKE_WORD on, only ones that open with a wake phrase count instead "what's the time" -> ask "what's the time" "Hey Bolt, what's the time" -> ask "what's the time" "Hey Bolt" -> chime, go to `command` command the next thing said (within COMMAND_WAIT_SEC) is the request follow_up the reply ended on a question: the next thing said (within FOLLOW_UP_SEC) is the answer, no wake phrase needed Speaking is half-duplex: the mic is ignored while Bolt talks and for a moment after, so his own voice through the speakers can't wake him (his replies say "Bolt" often enough). Transcribing is not: waiting for the server's transcript happens on a background thread, so the mic keeps being segmented meanwhile and whatever is said next becomes its own utterance and is sent off too. Finished transcripts are delivered oldest-first between mic frames. Everything that touches hardware or the network is injected, so the flow is tested with fakes; __main__ wires in the real mic, server and speaker. """ from __future__ import annotations import platform import queue import threading import time from collections import deque from typing import Callable, Optional from . import audio, config, server, speech_text from .listener import Segmenter from .stt import Transcription from .tools import Tools from .wake import find_wake ECHO_GUARD_SEC = 0.4 # ignore the mic this long after speaking def os_label() -> str: try: return f"{platform.system()} {platform.release()}".strip() or "a computer" except Exception: return "a computer" def annotate(text: str, commands: str) -> str: """The note on every request: it's voice, and what Bolt may do here.""" can = { "off": "You can't act on this computer from here.", "read": ('You can list and read files with your command tool, e.g. ' '`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real ' "output comes back. You can't run shell commands or change files from here."), "all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` ' "for files (list, read, write, edit) or `command: `; the real " "output comes back."), }[commands] return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was " "spoken and transcribed, so it may contain recognition mistakes. Your reply is read " "aloud: keep it short and conversational, with no markdown, lists, links or code. " f"{can} Never make up a command's output.]") class _PendingTranscript: """One utterance already sent off, its transcript still on the way.""" __slots__ = ("thread", "text", "error") def __init__(self): self.thread: Optional[threading.Thread] = None self.text = "" self.error: Optional[Exception] = None class Assistant: def __init__( self, *, transcription: Callable = Transcription, converse: Callable = server.converse, converse_stream: Optional[Callable] = server.converse_stream, tts: Callable = server.tts_stream, play: Callable = audio.play, tools: Optional[Tools] = None, log: Callable[[str], None] = print, clock: Callable[[], float] = time.monotonic, wake_words=None, anywhere_words=None, wake_required: Optional[bool] = None, verbose: bool = False, ): self._transcription = transcription self._converse = converse self._stream = converse_stream if config.STREAMING_REPLIES else None self._tts, self._play = tts, play self.log, self.clock, self.verbose = log, clock, verbose self.tools = tools or Tools(log=log) self.wake_words = wake_words or config.WAKE_WORDS self.anywhere_words = config.ANYWHERE_WAKE_WORDS if anywhere_words is None else anywhere_words self.wake_required = config.REQUIRE_WAKE_WORD if wake_required is None else wake_required self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES, config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC, config.MAX_UTTERANCE_SEC) self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000) # ~160 s of audio self.state = "idle" self._deadline = 0.0 self._follow_ups = 0 self._stt = None self._pending: "deque[_PendingTranscript]" = deque() self._mute_until = 0.0 self._voice = "" # a voice the server picked (speak_as) sticks for later replies self._tts_failed = False # ── audio in ───────────────────────────────────────────────────────── def on_mic_frame(self, frame: bytes) -> None: """Called from the audio thread: queue it, never block.""" try: self.frames.put_nowait(frame) except queue.Full: pass # we've fallen far behind; dropping audio beats unbounded memory def run(self, stop: threading.Event) -> None: self.log(self.listening_banner()) while not stop.is_set(): try: frame = self.frames.get(timeout=0.2) except queue.Empty: self._check_deadline() else: self.process(frame) self.handle_transcripts() def listening_banner(self) -> str: if not self.wake_required: return "Listening - anything you say is a request. Ctrl+C to quit." return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.' def process(self, frame: bytes) -> None: if self.clock() < self._mute_until: return self._check_deadline() for event in self.segmenter.feed(frame): if event.kind == "start": self._stt = self._transcription() self._stt.add(event.frames) elif event.kind == "frame" and self._stt is not None: self._stt.add(event.frames) elif event.kind == "dropped": if self._stt is not None: self._stt.cancel() self._stt = None elif event.kind == "end": stt, self._stt = self._stt, None if stt is not None: self._start_finish(stt, event.audio) def _start_finish(self, stt, utterance: bytes) -> None: """finish() waits on the server; done here, that wait would deafen the loop. Run it on a thread and keep segmenting the mic meanwhile.""" pending = _PendingTranscript() def work() -> None: try: pending.text = stt.finish(utterance) except Exception as exc: # SttError, or the socket misbehaving pending.error = exc pending.thread = threading.Thread(target=work, name="stt-finish", daemon=True) self._pending.append(pending) pending.thread.start() def handle_transcripts(self, wait: bool = False) -> None: """Deliver finished transcripts, oldest first. One still in flight — or an utterance still being spoken — ends the pass rather than blocking, unless *wait* (tests, shutdown). The head stays queued while heard() runs so _check_deadline knows speech already happened.""" while self._pending: if self.segmenter.in_speech and not wait: return head = self._pending[0] if wait: head.thread.join() if head.thread.is_alive(): return if head.error is not None: self._pending.popleft() self.log(f"! transcription failed: {head.error}") if self.state != "idle": self._chime(audio.ERROR_CHIME) continue try: self.heard(head.text) finally: self._pending.popleft() def _check_deadline(self) -> None: # A pending transcript means something *was* said in time — it's # just still at the server; the window mustn't close on that latency. if self.state in ("command", "follow_up") and self.clock() > self._deadline \ and not self.segmenter.in_speech and not self._pending: if self.verbose: self.log(" (stopped waiting)") self.state = "idle" # ── what was said ──────────────────────────────────────────────────── def heard(self, text: str) -> None: text = (text or "").strip() if not text: return self._check_deadline() woke, rest = find_wake(text, self.wake_words, self.anywhere_words) if self.state == "idle": if not self.wake_required and not woke: # Always listening: the mic being open is the activation, so # whatever it managed to hear is the request. self._follow_ups = 0 self.ask(text) return if not woke: if self.verbose: self.log(f" (not for me) {text}") return if not rest: self.log("* Yes?") self._chime(audio.WAKE_CHIME) self._wait("command", config.COMMAND_WAIT_SEC) return self._follow_ups = 0 self.ask(rest) return # command / follow_up: whatever was said is the request. A repeated # "Hey Bolt" in front of it is dropped; on its own it just re-arms. if woke and not rest: self._chime(audio.WAKE_CHIME) self._wait(self.state, config.COMMAND_WAIT_SEC) return if self.state == "command": self._follow_ups = 0 self.ask(rest if woke else text) def _wait(self, state: str, seconds: float) -> None: self.state = state self._deadline = self.clock() + seconds # ── asking and answering ───────────────────────────────────────────── def ask(self, request: str) -> None: self.state = "busy" self.log(f"> {request}") spoken = [] def say(sentence: str) -> None: spoken.append(sentence) self.speak(sentence) message = annotate(request, self.tools.mode) try: reply = None if self._stream is not None: try: reply = self._stream(message, on_say=say, on_command=self.tools) except server.ServerError: reply = None # nothing was said yet: the plain path is invisible if reply is None: reply = self._converse(message, on_command=self.tools, on_say=say) except server.ServerError as exc: self.log(f"! {exc}") self._chime(audio.ERROR_CHIME) self._back_to_listening("idle") return if reply.voice_id: self._voice = reply.voice_id if not reply.streamed and reply.text.strip(): spoken.append(reply.text) self.speak(reply.text) full = " ".join(spoken) if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS: self._follow_ups += 1 self._back_to_listening("follow_up", config.FOLLOW_UP_SEC) else: self._follow_ups = 0 self._back_to_listening("idle") def speak(self, text: str) -> None: words = speech_text.for_speech(text) if not words: return self.log(f"< {speech_text.for_display(text)}") try: self._play(self._tts(words, self._voice)) self._tts_failed = False except server.TtsError as exc: if not self._tts_failed: # say it once, not once per sentence self.log(f"! can't speak: {exc}") self._tts_failed = True except Exception as exc: self.log(f"! playback failed: {exc}") def _chime(self, sound: bytes) -> None: if not config.CHIMES: return try: self._play([sound]) except Exception: pass self._mute_until = self.clock() + ECHO_GUARD_SEC def _back_to_listening(self, state: str, seconds: float = 0.0) -> None: """Drop what the mic picked up while Bolt was talking (his own voice), start detection fresh, and give the room a moment to go quiet.""" while True: try: self.frames.get_nowait() except queue.Empty: break self.segmenter.reset() if self._stt is not None: self._stt.cancel() self._stt = None self._mute_until = self.clock() + ECHO_GUARD_SEC self._wait(state, seconds + ECHO_GUARD_SEC)