"""The assistant: hear the wake phrase, take the request, answer out loud. idle every utterance the local detector passes on is transcribed by the server; only ones that open with a wake phrase count "Hey Bolt, what's the time" -> ask "what's the time" "Hey Bolt" -> chime, go to `command` command the next thing said (within COMMAND_WAIT_SEC) is the request follow_up the reply ended on a question: the next thing said (within FOLLOW_UP_SEC) is the answer, no wake phrase needed Speaking is half-duplex: the mic is ignored while Bolt talks and for a moment after, so his own voice through the speakers can't wake him (his replies say "Bolt" often enough). Everything that touches hardware or the network is injected, so the flow is tested with fakes; __main__ wires in the real mic, server and speaker. """ from __future__ import annotations import platform import queue import threading import time from typing import Callable, Optional from . import audio, config, server, speech_text from .listener import Segmenter from .stt import SttError, Transcription from .tools import Tools from .wake import find_wake ECHO_GUARD_SEC = 0.4 # ignore the mic this long after speaking def os_label() -> str: try: return f"{platform.system()} {platform.release()}".strip() or "a computer" except Exception: return "a computer" def annotate(text: str, commands: str) -> str: """The note on every request: it's voice, and what Bolt may do here.""" can = { "off": "You can't act on this computer from here.", "read": ('You can list and read files with your command tool, e.g. ' '`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real ' "output comes back. You can't run shell commands or change files from here."), "all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` ' "for files (list, read, write, edit) or `command: `; the real " "output comes back."), }[commands] return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was " "spoken and transcribed, so it may contain recognition mistakes. Your reply is read " "aloud: keep it short and conversational, with no markdown, lists, links or code. " f"{can} Never make up a command's output.]") class Assistant: def __init__( self, *, transcription: Callable = Transcription, converse: Callable = server.converse, converse_stream: Optional[Callable] = server.converse_stream, tts: Callable = server.tts_stream, play: Callable = audio.play, tools: Optional[Tools] = None, log: Callable[[str], None] = print, clock: Callable[[], float] = time.monotonic, wake_words=None, anywhere_words=None, verbose: bool = False, ): self._transcription = transcription self._converse = converse self._stream = converse_stream if config.STREAMING_REPLIES else None self._tts, self._play = tts, play self.log, self.clock, self.verbose = log, clock, verbose self.tools = tools or Tools(log=log) self.wake_words = wake_words or config.WAKE_WORDS self.anywhere_words = config.ANYWHERE_WAKE_WORDS if anywhere_words is None else anywhere_words self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES, config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC, config.MAX_UTTERANCE_SEC) self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000) # ~160 s of audio self.state = "idle" self._deadline = 0.0 self._follow_ups = 0 self._stt = None self._mute_until = 0.0 self._voice = "" # a voice the server picked (speak_as) sticks for later replies self._tts_failed = False # ── audio in ───────────────────────────────────────────────────────── def on_mic_frame(self, frame: bytes) -> None: """Called from the audio thread: queue it, never block.""" try: self.frames.put_nowait(frame) except queue.Full: pass # we've fallen far behind; dropping audio beats unbounded memory def run(self, stop: threading.Event) -> None: self.log(self.listening_banner()) while not stop.is_set(): try: frame = self.frames.get(timeout=0.2) except queue.Empty: self._check_deadline() continue self.process(frame) def listening_banner(self) -> str: return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.' def process(self, frame: bytes) -> None: if self.clock() < self._mute_until: return self._check_deadline() for event in self.segmenter.feed(frame): if event.kind == "start": self._stt = self._transcription() self._stt.add(event.frames) elif event.kind == "frame" and self._stt is not None: self._stt.add(event.frames) elif event.kind == "dropped": if self._stt is not None: self._stt.cancel() self._stt = None elif event.kind == "end": stt, self._stt = self._stt, None try: text = stt.finish(event.audio) if stt is not None else "" except SttError as exc: self.log(f"! transcription failed: {exc}") if self.state != "idle": self._chime(audio.ERROR_CHIME) continue self.heard(text) def _check_deadline(self) -> None: if self.state in ("command", "follow_up") and self.clock() > self._deadline \ and not self.segmenter.in_speech: if self.verbose: self.log(" (stopped waiting)") self.state = "idle" # ── what was said ──────────────────────────────────────────────────── def heard(self, text: str) -> None: text = (text or "").strip() if not text: return self._check_deadline() woke, rest = find_wake(text, self.wake_words, self.anywhere_words) if self.state == "idle": if not woke: if self.verbose: self.log(f" (not for me) {text}") return if not rest: self.log("* Yes?") self._chime(audio.WAKE_CHIME) self._wait("command", config.COMMAND_WAIT_SEC) return self._follow_ups = 0 self.ask(rest) return # command / follow_up: whatever was said is the request. A repeated # "Hey Bolt" in front of it is dropped; on its own it just re-arms. if woke and not rest: self._chime(audio.WAKE_CHIME) self._wait(self.state, config.COMMAND_WAIT_SEC) return if self.state == "command": self._follow_ups = 0 self.ask(rest if woke else text) def _wait(self, state: str, seconds: float) -> None: self.state = state self._deadline = self.clock() + seconds # ── asking and answering ───────────────────────────────────────────── def ask(self, request: str) -> None: self.state = "busy" self.log(f"> {request}") spoken = [] def say(sentence: str) -> None: spoken.append(sentence) self.speak(sentence) message = annotate(request, self.tools.mode) try: reply = None if self._stream is not None: try: reply = self._stream(message, on_say=say, on_command=self.tools) except server.ServerError: reply = None # nothing was said yet: the plain path is invisible if reply is None: reply = self._converse(message, on_command=self.tools, on_say=say) except server.ServerError as exc: self.log(f"! {exc}") self._chime(audio.ERROR_CHIME) self._back_to_listening("idle") return if reply.voice_id: self._voice = reply.voice_id if not reply.streamed and reply.text.strip(): spoken.append(reply.text) self.speak(reply.text) full = " ".join(spoken) if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS: self._follow_ups += 1 self._back_to_listening("follow_up", config.FOLLOW_UP_SEC) else: self._follow_ups = 0 self._back_to_listening("idle") def speak(self, text: str) -> None: words = speech_text.for_speech(text) if not words: return self.log(f"< {speech_text.for_display(text)}") try: self._play(self._tts(words, self._voice)) self._tts_failed = False except server.TtsError as exc: if not self._tts_failed: # say it once, not once per sentence self.log(f"! can't speak: {exc}") self._tts_failed = True except Exception as exc: self.log(f"! playback failed: {exc}") def _chime(self, sound: bytes) -> None: if not config.CHIMES: return try: self._play([sound]) except Exception: pass self._mute_until = self.clock() + ECHO_GUARD_SEC def _back_to_listening(self, state: str, seconds: float = 0.0) -> None: """Drop what the mic picked up while Bolt was talking (his own voice), start detection fresh, and give the room a moment to go quiet.""" while True: try: self.frames.get_nowait() except queue.Empty: break self.segmenter.reset() if self._stt is not None: self._stt.cancel() self._stt = None self._mute_until = self.clock() + ECHO_GUARD_SEC self._wait(state, seconds + ECHO_GUARD_SEC)