Bolt-Assistant/bolt_assistant/assistant.py

"""The assistant: hear a request, answer out loud.

    idle      every utterance the local detector passes on is transcribed by
              the server and, by default, is itself the request — the mic
              being open is the activation. With REQUIRE_WAKE_WORD on, only
              ones that open with a wake phrase count instead
                "what's the time"            -> ask "what's the time"
                "Hey Bolt, what's the time"  -> ask "what's the time"
                "Hey Bolt"                   -> chime, go to `command`
    command   the next thing said (within COMMAND_WAIT_SEC) is the request
    follow_up the reply ended on a question: the next thing said (within
              FOLLOW_UP_SEC) is the answer, no wake phrase needed

Speaking is half-duplex: the mic is ignored while Bolt talks and for a
moment after, so his own voice through the speakers can't wake him (his
replies say "Bolt" often enough).

Transcribing is not: waiting for the server's transcript happens on a
background thread, so the mic keeps being segmented meanwhile and whatever
is said next becomes its own utterance and is sent off too. Finished
transcripts are delivered oldest-first between mic frames.

Announcements (Bolt's reminders, delivered-file notices) ride the heartbeat
thread and wait in a queue; the run loop speaks them one at a time whenever
it is idle, then flushes the echo of Bolt's own voice like after a reply.

Everything that touches hardware or the network is injected, so the flow
is tested with fakes; __main__ wires in the real mic, server and speaker.
"""

from __future__ import annotations

import platform
import queue
import threading
import time
from collections import deque
from typing import Callable, Optional

from . import audio, config, server, speech_text
from .listener import Segmenter
from .stt import Transcription
from .tools import Tools
from .wake import find_wake

ECHO_GUARD_SEC = 0.4  # ignore the mic this long after speaking


def os_label() -> str:
    try:
        return f"{platform.system()} {platform.release()}".strip() or "a computer"
    except Exception:
        return "a computer"


def annotate(text: str, commands: str) -> str:
    """The note on every request: it's voice, and what Bolt may do here."""
    can = {
        "off": "You can't act on this computer from here.",
        "read": ('You can list and read files with your command tool, e.g. '
                 '`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real '
                 "output comes back. You can't run shell commands or change files from here."),
        "all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` '
                "for files (list, read, write, edit) or `command: <shell command>`; the real "
                "output comes back."),
    }[commands]
    return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was "
            "spoken and transcribed, so it may contain recognition mistakes. Your reply is read "
            "aloud: keep it short and conversational, with no markdown, lists, links or code. "
            f"{can} Never make up a command's output.]")


class _PendingTranscript:
    """One utterance already sent off, its transcript still on the way."""

    __slots__ = ("thread", "text", "error")

    def __init__(self):
        self.thread: Optional[threading.Thread] = None
        self.text = ""
        self.error: Optional[Exception] = None


class Assistant:
    def __init__(
        self,
        *,
        transcription: Callable = Transcription,
        converse: Callable = server.converse,
        converse_stream: Optional[Callable] = server.converse_stream,
        tts: Callable = server.tts_stream,
        play: Callable = audio.play,
        tools: Optional[Tools] = None,
        log: Callable[[str], None] = print,
        clock: Callable[[], float] = time.monotonic,
        wake_words=None,
        anywhere_words=None,
        wake_required: Optional[bool] = None,
        verbose: bool = False,
    ):
        self._transcription = transcription
        self._converse = converse
        self._stream = converse_stream if config.STREAMING_REPLIES else None
        self._tts, self._play = tts, play
        self.log, self.clock, self.verbose = log, clock, verbose
        self.tools = tools or Tools(log=log)
        self.wake_words = wake_words or config.WAKE_WORDS
        self.anywhere_words = config.ANYWHERE_WAKE_WORDS if anywhere_words is None else anywhere_words
        self.wake_required = config.REQUIRE_WAKE_WORD if wake_required is None else wake_required
        self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES,
                                   config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC,
                                   config.MAX_UTTERANCE_SEC)
        self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000)  # ~160 s of audio
        self.state = "idle"
        self._deadline = 0.0
        self._follow_ups = 0
        self._stt = None
        self._pending: "deque[_PendingTranscript]" = deque()
        self._mute_until = 0.0
        self._voice = ""  # a voice the server picked (speak_as) sticks for later replies
        self._tts_failed = False
        self._announcements: "queue.Queue[str]" = queue.Queue()  # heartbeat notices, spoken when idle

    # ── audio in ─────────────────────────────────────────────────────────

    def on_mic_frame(self, frame: bytes) -> None:
        """Called from the audio thread: queue it, never block."""
        try:
            self.frames.put_nowait(frame)
        except queue.Full:
            pass  # we've fallen far behind; dropping audio beats unbounded memory

    def run(self, stop: threading.Event) -> None:
        self.log(self.listening_banner())
        while not stop.is_set():
            try:
                frame = self.frames.get(timeout=0.2)
            except queue.Empty:
                self._check_deadline()
            else:
                self.process(frame)
            self.handle_transcripts()
            self._drain_announcements()

    def listening_banner(self) -> str:
        if not self.wake_required:
            return "Listening - anything you say is a request. Ctrl+C to quit."
        return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.'

    def process(self, frame: bytes) -> None:
        if self.clock() < self._mute_until:
            return
        self._check_deadline()
        for event in self.segmenter.feed(frame):
            if event.kind == "start":
                self._stt = self._transcription()
                self._stt.add(event.frames)
            elif event.kind == "frame" and self._stt is not None:
                self._stt.add(event.frames)
            elif event.kind == "dropped":
                if self._stt is not None:
                    self._stt.cancel()
                self._stt = None
            elif event.kind == "end":
                stt, self._stt = self._stt, None
                if stt is not None:
                    self._start_finish(stt, event.audio)

    def _start_finish(self, stt, utterance: bytes) -> None:
        """finish() waits on the server; done here, that wait would deafen
        the loop. Run it on a thread and keep segmenting the mic meanwhile."""
        pending = _PendingTranscript()

        def work() -> None:
            try:
                pending.text = stt.finish(utterance)
            except Exception as exc:  # SttError, or the socket misbehaving
                pending.error = exc

        pending.thread = threading.Thread(target=work, name="stt-finish", daemon=True)
        self._pending.append(pending)
        pending.thread.start()

    def handle_transcripts(self, wait: bool = False) -> None:
        """Deliver finished transcripts, oldest first. One still in flight —
        or an utterance still being spoken — ends the pass rather than
        blocking, unless *wait* (tests, shutdown). The head stays queued
        while heard() runs so _check_deadline knows speech already happened."""
        while self._pending:
            if self.segmenter.in_speech and not wait:
                return
            head = self._pending[0]
            if wait:
                head.thread.join()
            if head.thread.is_alive():
                return
            if head.error is not None:
                self._pending.popleft()
                self.log(f"! transcription failed: {head.error}")
                if self.state != "idle":
                    self._chime(audio.ERROR_CHIME)
                continue
            try:
                self.heard(head.text)
            finally:
                self._pending.popleft()

    def _check_deadline(self) -> None:
        # A pending transcript means something *was* said in time — it's
        # just still at the server; the window mustn't close on that latency.
        if self.state in ("command", "follow_up") and self.clock() > self._deadline \
                and not self.segmenter.in_speech and not self._pending:
            if self.verbose:
                self.log("  (stopped waiting)")
            self.state = "idle"

    # ── heartbeat announcements ──────────────────────────────────────────

    def announce(self, text: str) -> None:
        """Queue something to say once the room is free — a reminder Bolt
        fired, a delivered-file notice. Never interrupts: it waits for idle."""
        text = (text or "").strip()
        if text:
            self._announcements.put(text)

    def _drain_announcements(self) -> None:
        """Say what the heartbeat brought, one at a time, while idle — never
        mid-sentence, while a transcript is still in flight, or during the
        echo guard after Bolt's own voice."""
        if (self.state != "idle" or self._pending or self.segmenter.in_speech
                or self.clock() < self._mute_until):
            return
        try:
            text = self._announcements.get_nowait()
        except queue.Empty:
            return
        self.speak(text)
        self._back_to_listening("idle")  # flush the mic's echo of Bolt's own voice

    # ── what was said ────────────────────────────────────────────────────

    def heard(self, text: str) -> None:
        text = (text or "").strip()
        if not text:
            return
        self._check_deadline()
        woke, rest = find_wake(text, self.wake_words, self.anywhere_words)
        if self.state == "idle":
            if not self.wake_required and not woke:
                # Always listening: the mic being open is the activation, so
                # whatever it managed to hear is the request.
                self._follow_ups = 0
                self.ask(text)
                return
            if not woke:
                if self.verbose:
                    self.log(f"  (not for me) {text}")
                return
            if not rest:
                self.log("* Yes?")
                self._chime(audio.WAKE_CHIME)
                self._wait("command", config.COMMAND_WAIT_SEC)
                return
            self._follow_ups = 0
            self.ask(rest)
            return
        # command / follow_up: whatever was said is the request. A repeated
        # "Hey Bolt" in front of it is dropped; on its own it just re-arms.
        if woke and not rest:
            self._chime(audio.WAKE_CHIME)
            self._wait(self.state, config.COMMAND_WAIT_SEC)
            return
        if self.state == "command":
            self._follow_ups = 0
        self.ask(rest if woke else text)

    def _wait(self, state: str, seconds: float) -> None:
        self.state = state
        self._deadline = self.clock() + seconds

    # ── asking and answering ─────────────────────────────────────────────

    def ask(self, request: str) -> None:
        self.state = "busy"
        self.log(f"> {request}")
        spoken = []

        def say(sentence: str) -> None:
            spoken.append(sentence)
            self.speak(sentence)

        message = annotate(request, self.tools.mode)
        try:
            reply = None
            if self._stream is not None:
                try:
                    reply = self._stream(message, on_say=say, on_command=self.tools)
                except server.ServerError:
                    reply = None  # nothing was said yet: the plain path is invisible
            if reply is None:
                reply = self._converse(message, on_command=self.tools, on_say=say)
        except server.ServerError as exc:
            self.log(f"! {exc}")
            self._chime(audio.ERROR_CHIME)
            self._back_to_listening("idle")
            return
        if reply.voice_id:
            self._voice = reply.voice_id
        if not reply.streamed and reply.text.strip():
            spoken.append(reply.text)
            self.speak(reply.text)
        full = " ".join(spoken)
        if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS:
            self._follow_ups += 1
            self._back_to_listening("follow_up", config.FOLLOW_UP_SEC)
        else:
            self._follow_ups = 0
            self._back_to_listening("idle")

    def speak(self, text: str) -> None:
        words = speech_text.for_speech(text)
        if not words:
            return
        self.log(f"< {speech_text.for_display(text)}")
        try:
            self._play(self._tts(words, self._voice))
            self._tts_failed = False
        except server.TtsError as exc:
            if not self._tts_failed:  # say it once, not once per sentence
                self.log(f"! can't speak: {exc}")
            self._tts_failed = True
        except Exception as exc:
            self.log(f"! playback failed: {exc}")

    def _chime(self, sound: bytes) -> None:
        if not config.CHIMES:
            return
        try:
            self._play([sound])
        except Exception:
            pass
        self._mute_until = self.clock() + ECHO_GUARD_SEC

    def _back_to_listening(self, state: str, seconds: float = 0.0) -> None:
        """Drop what the mic picked up while Bolt was talking (his own voice),
        start detection fresh, and give the room a moment to go quiet."""
        while True:
            try:
                self.frames.get_nowait()
            except queue.Empty:
                break
        self.segmenter.reset()
        if self._stt is not None:
            self._stt.cancel()
            self._stt = None
        self._mute_until = self.clock() + ECHO_GUARD_SEC
        self._wait(state, seconds + ECHO_GUARD_SEC)