Bolt Assistant: wake-phrase voice assistant for Bolt
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
"""The assistant: hear the wake phrase, take the request, answer out loud.
|
||||
|
||||
idle every utterance the local detector passes on is transcribed by
|
||||
the server; only ones that open with a wake phrase count
|
||||
"Hey Bolt, what's the time" -> ask "what's the time"
|
||||
"Hey Bolt" -> chime, go to `command`
|
||||
command the next thing said (within COMMAND_WAIT_SEC) is the request
|
||||
follow_up the reply ended on a question: the next thing said (within
|
||||
FOLLOW_UP_SEC) is the answer, no wake phrase needed
|
||||
|
||||
Speaking is half-duplex: the mic is ignored while Bolt talks and for a
|
||||
moment after, so his own voice through the speakers can't wake him (his
|
||||
replies say "Bolt" often enough).
|
||||
|
||||
Everything that touches hardware or the network is injected, so the flow
|
||||
is tested with fakes; __main__ wires in the real mic, server and speaker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
from . import audio, config, server, speech_text
|
||||
from .listener import Segmenter
|
||||
from .stt import SttError, Transcription
|
||||
from .tools import Tools
|
||||
from .wake import find_wake
|
||||
|
||||
ECHO_GUARD_SEC = 0.4 # ignore the mic this long after speaking
|
||||
|
||||
|
||||
def os_label() -> str:
|
||||
try:
|
||||
return f"{platform.system()} {platform.release()}".strip() or "a computer"
|
||||
except Exception:
|
||||
return "a computer"
|
||||
|
||||
|
||||
def annotate(text: str, commands: str) -> str:
|
||||
"""The note on every request: it's voice, and what Bolt may do here."""
|
||||
can = {
|
||||
"off": "You can't act on this computer from here.",
|
||||
"read": ('You can list and read files with your command tool, e.g. '
|
||||
'`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real '
|
||||
"output comes back. You can't run shell commands or change files from here."),
|
||||
"all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` '
|
||||
"for files (list, read, write, edit) or `command: <shell command>`; the real "
|
||||
"output comes back."),
|
||||
}[commands]
|
||||
return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was "
|
||||
"spoken and transcribed, so it may contain recognition mistakes. Your reply is read "
|
||||
"aloud: keep it short and conversational, with no markdown, lists, links or code. "
|
||||
f"{can} Never make up a command's output.]")
|
||||
|
||||
|
||||
class Assistant:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
transcription: Callable = Transcription,
|
||||
converse: Callable = server.converse,
|
||||
converse_stream: Optional[Callable] = server.converse_stream,
|
||||
tts: Callable = server.tts_stream,
|
||||
play: Callable = audio.play,
|
||||
tools: Optional[Tools] = None,
|
||||
log: Callable[[str], None] = print,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
wake_words=None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
self._transcription = transcription
|
||||
self._converse = converse
|
||||
self._stream = converse_stream if config.STREAMING_REPLIES else None
|
||||
self._tts, self._play = tts, play
|
||||
self.log, self.clock, self.verbose = log, clock, verbose
|
||||
self.tools = tools or Tools(log=log)
|
||||
self.wake_words = wake_words or config.WAKE_WORDS
|
||||
self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES,
|
||||
config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC,
|
||||
config.MAX_UTTERANCE_SEC)
|
||||
self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000) # ~160 s of audio
|
||||
self.state = "idle"
|
||||
self._deadline = 0.0
|
||||
self._follow_ups = 0
|
||||
self._stt = None
|
||||
self._mute_until = 0.0
|
||||
self._voice = "" # a voice the server picked (speak_as) sticks for later replies
|
||||
self._tts_failed = False
|
||||
|
||||
# ── audio in ─────────────────────────────────────────────────────────
|
||||
|
||||
def on_mic_frame(self, frame: bytes) -> None:
|
||||
"""Called from the audio thread: queue it, never block."""
|
||||
try:
|
||||
self.frames.put_nowait(frame)
|
||||
except queue.Full:
|
||||
pass # we've fallen far behind; dropping audio beats unbounded memory
|
||||
|
||||
def run(self, stop: threading.Event) -> None:
|
||||
self.log(self.listening_banner())
|
||||
while not stop.is_set():
|
||||
try:
|
||||
frame = self.frames.get(timeout=0.2)
|
||||
except queue.Empty:
|
||||
self._check_deadline()
|
||||
continue
|
||||
self.process(frame)
|
||||
|
||||
def listening_banner(self) -> str:
|
||||
return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.'
|
||||
|
||||
def process(self, frame: bytes) -> None:
|
||||
if self.clock() < self._mute_until:
|
||||
return
|
||||
self._check_deadline()
|
||||
for event in self.segmenter.feed(frame):
|
||||
if event.kind == "start":
|
||||
self._stt = self._transcription()
|
||||
self._stt.add(event.frames)
|
||||
elif event.kind == "frame" and self._stt is not None:
|
||||
self._stt.add(event.frames)
|
||||
elif event.kind == "dropped":
|
||||
if self._stt is not None:
|
||||
self._stt.cancel()
|
||||
self._stt = None
|
||||
elif event.kind == "end":
|
||||
stt, self._stt = self._stt, None
|
||||
try:
|
||||
text = stt.finish(event.audio) if stt is not None else ""
|
||||
except SttError as exc:
|
||||
self.log(f"! transcription failed: {exc}")
|
||||
if self.state != "idle":
|
||||
self._chime(audio.ERROR_CHIME)
|
||||
continue
|
||||
self.heard(text)
|
||||
|
||||
def _check_deadline(self) -> None:
|
||||
if self.state in ("command", "follow_up") and self.clock() > self._deadline \
|
||||
and not self.segmenter.in_speech:
|
||||
if self.verbose:
|
||||
self.log(" (stopped waiting)")
|
||||
self.state = "idle"
|
||||
|
||||
# ── what was said ────────────────────────────────────────────────────
|
||||
|
||||
def heard(self, text: str) -> None:
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return
|
||||
self._check_deadline()
|
||||
woke, rest = find_wake(text, self.wake_words)
|
||||
if self.state == "idle":
|
||||
if not woke:
|
||||
if self.verbose:
|
||||
self.log(f" (not for me) {text}")
|
||||
return
|
||||
if not rest:
|
||||
self.log("* Yes?")
|
||||
self._chime(audio.WAKE_CHIME)
|
||||
self._wait("command", config.COMMAND_WAIT_SEC)
|
||||
return
|
||||
self._follow_ups = 0
|
||||
self.ask(rest)
|
||||
return
|
||||
# command / follow_up: whatever was said is the request. A repeated
|
||||
# "Hey Bolt" in front of it is dropped; on its own it just re-arms.
|
||||
if woke and not rest:
|
||||
self._chime(audio.WAKE_CHIME)
|
||||
self._wait(self.state, config.COMMAND_WAIT_SEC)
|
||||
return
|
||||
if self.state == "command":
|
||||
self._follow_ups = 0
|
||||
self.ask(rest if woke else text)
|
||||
|
||||
def _wait(self, state: str, seconds: float) -> None:
|
||||
self.state = state
|
||||
self._deadline = self.clock() + seconds
|
||||
|
||||
# ── asking and answering ─────────────────────────────────────────────
|
||||
|
||||
def ask(self, request: str) -> None:
|
||||
self.state = "busy"
|
||||
self.log(f"> {request}")
|
||||
spoken = []
|
||||
|
||||
def say(sentence: str) -> None:
|
||||
spoken.append(sentence)
|
||||
self.speak(sentence)
|
||||
|
||||
message = annotate(request, self.tools.mode)
|
||||
try:
|
||||
reply = None
|
||||
if self._stream is not None:
|
||||
try:
|
||||
reply = self._stream(message, on_say=say, on_command=self.tools)
|
||||
except server.ServerError:
|
||||
reply = None # nothing was said yet: the plain path is invisible
|
||||
if reply is None:
|
||||
reply = self._converse(message, on_command=self.tools, on_say=say)
|
||||
except server.ServerError as exc:
|
||||
self.log(f"! {exc}")
|
||||
self._chime(audio.ERROR_CHIME)
|
||||
self._back_to_listening("idle")
|
||||
return
|
||||
if reply.voice_id:
|
||||
self._voice = reply.voice_id
|
||||
if not reply.streamed and reply.text.strip():
|
||||
spoken.append(reply.text)
|
||||
self.speak(reply.text)
|
||||
full = " ".join(spoken)
|
||||
if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS:
|
||||
self._follow_ups += 1
|
||||
self._back_to_listening("follow_up", config.FOLLOW_UP_SEC)
|
||||
else:
|
||||
self._follow_ups = 0
|
||||
self._back_to_listening("idle")
|
||||
|
||||
def speak(self, text: str) -> None:
|
||||
words = speech_text.for_speech(text)
|
||||
if not words:
|
||||
return
|
||||
self.log(f"< {speech_text.for_display(text)}")
|
||||
try:
|
||||
self._play(self._tts(words, self._voice))
|
||||
self._tts_failed = False
|
||||
except server.TtsError as exc:
|
||||
if not self._tts_failed: # say it once, not once per sentence
|
||||
self.log(f"! can't speak: {exc}")
|
||||
self._tts_failed = True
|
||||
except Exception as exc:
|
||||
self.log(f"! playback failed: {exc}")
|
||||
|
||||
def _chime(self, sound: bytes) -> None:
|
||||
if not config.CHIMES:
|
||||
return
|
||||
try:
|
||||
self._play([sound])
|
||||
except Exception:
|
||||
pass
|
||||
self._mute_until = self.clock() + ECHO_GUARD_SEC
|
||||
|
||||
def _back_to_listening(self, state: str, seconds: float = 0.0) -> None:
|
||||
"""Drop what the mic picked up while Bolt was talking (his own voice),
|
||||
start detection fresh, and give the room a moment to go quiet."""
|
||||
while True:
|
||||
try:
|
||||
self.frames.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self.segmenter.reset()
|
||||
if self._stt is not None:
|
||||
self._stt.cancel()
|
||||
self._stt = None
|
||||
self._mute_until = self.clock() + ECHO_GUARD_SEC
|
||||
self._wait(state, seconds + ECHO_GUARD_SEC)
|
||||
Reference in New Issue
Block a user