From 34b2d302f9c46e4f72fd0c119345e020c6737327 Mon Sep 17 00:00:00 2001 From: TMN AI Date: Wed, 30 Sep 2026 20:25:15 -0600 Subject: [PATCH] Updates to the assistants overall abilities, like always listening, and only responding on messages when activated. --- .claude/worktrees/always-send | 1 + .claude/worktrees/wake-phrase-mic-fix | 1 + .env.example | 2 ++ README.md | 2 ++ bolt_assistant/assistant.py | 4 ++- bolt_assistant/audio.py | 37 ++++++++++++++++++++++++++- bolt_assistant/config.py | 4 +++ bolt_assistant/wake.py | 23 ++++++++++++++--- run.sh | 0 tests/test_assistant.py | 15 ++++++++++- 10 files changed, 82 insertions(+), 7 deletions(-) create mode 160000 .claude/worktrees/always-send create mode 160000 .claude/worktrees/wake-phrase-mic-fix mode change 100644 => 100755 run.sh diff --git a/.claude/worktrees/always-send b/.claude/worktrees/always-send new file mode 160000 index 0000000..8bcd539 --- /dev/null +++ b/.claude/worktrees/always-send @@ -0,0 +1 @@ +Subproject commit 8bcd539b3107fb090b4ad7cedc07e4fa6c220a4f diff --git a/.claude/worktrees/wake-phrase-mic-fix b/.claude/worktrees/wake-phrase-mic-fix new file mode 160000 index 0000000..2f53bf3 --- /dev/null +++ b/.claude/worktrees/wake-phrase-mic-fix @@ -0,0 +1 @@ +Subproject commit 2f53bf36857ba742c331e25086546a9367f50438 diff --git a/.env.example b/.env.example index 575d47f..3ea4fd2 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,8 @@ ELEVENLABS_VOICE_ID= # Wake phrases, comma-separated (matched on the transcript, at the start of what's said) #WAKE_WORDS=hey bolt,bolt,thunderbolt,hey thunderbolt,ok bolt,okay bolt,hey assistant +# Words that count anywhere in what's said; the whole sentence becomes the request +#ANYWHERE_WAKE_WORDS=assistant # What Bolt may do on this machine when he relays a command: # off | read (default: list and read files) | all (shell commands and file changes too) diff --git a/README.md b/README.md index 6a9941d..238c6a4 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ See `.env.example`. The ones you're most likely to change: - `VAD_RMS_THRESHOLD`: raise it if background noise keeps getting transcribed, lower it if you have to raise your voice. - `WAKE_WORDS`: the phrases, comma-separated. +- `ANYWHERE_WAKE_WORDS`: words that count anywhere in a sentence, not just at the start + (default `assistant`: "what's the weather, assistant?"). Blank to turn off. - `COMMAND_WAIT_SEC`, `FOLLOW_UP_SEC`: how long it waits after "Hey Bolt", or after asking you something. diff --git a/bolt_assistant/assistant.py b/bolt_assistant/assistant.py index 954aaf5..861cd2b 100644 --- a/bolt_assistant/assistant.py +++ b/bolt_assistant/assistant.py @@ -70,6 +70,7 @@ class Assistant: log: Callable[[str], None] = print, clock: Callable[[], float] = time.monotonic, wake_words=None, + anywhere_words=None, verbose: bool = False, ): self._transcription = transcription @@ -79,6 +80,7 @@ class Assistant: self.log, self.clock, self.verbose = log, clock, verbose self.tools = tools or Tools(log=log) self.wake_words = wake_words or config.WAKE_WORDS + self.anywhere_words = config.ANYWHERE_WAKE_WORDS if anywhere_words is None else anywhere_words self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES, config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC, config.MAX_UTTERANCE_SEC) @@ -152,7 +154,7 @@ class Assistant: if not text: return self._check_deadline() - woke, rest = find_wake(text, self.wake_words) + woke, rest = find_wake(text, self.wake_words, self.anywhere_words) if self.state == "idle": if not woke: if self.verbose: diff --git a/bolt_assistant/audio.py b/bolt_assistant/audio.py index ea9b813..83ac7f2 100644 --- a/bolt_assistant/audio.py +++ b/bolt_assistant/audio.py @@ -30,7 +30,42 @@ def open_mic(on_frame: Callable[[bytes], None]): def play(chunks: Iterable[bytes], sample_rate: int = config.TTS_SAMPLE_RATE, should_stop: Callable[[], bool] = lambda: False) -> bool: - """Play PCM16 mono chunks as they arrive. False if stopped early.""" + """Play PCM16 mono chunks as they arrive. False if stopped early. + + Through `pacat`, not sounddevice: PortAudio only sees raw ALSA devices, + so a Bluetooth or PipeWire sink is invisible to it and the audio goes + somewhere nobody can hear. pacat talks to the audio server, so it + follows the system's default sink like every other player. + """ + import shutil + import subprocess + + if shutil.which("pacat") is None: + return _play_sounddevice(chunks, sample_rate, should_stop) + + proc = subprocess.Popen( + ["pacat", "--rate", str(sample_rate), "--channels", "1", + "--format", "s16le", "--latency-msec", "60"], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + for chunk in chunks: + if should_stop(): + return False + proc.stdin.write(chunk) + proc.stdin.close() + proc.wait(timeout=10) + except (BrokenPipeError, OSError): + return False + finally: + if proc.poll() is None: + proc.kill() + return True + + +def _play_sounddevice(chunks: Iterable[bytes], sample_rate: int, + should_stop: Callable[[], bool]) -> bool: + """Fallback for a machine without pacat.""" import sounddevice as sd with sd.RawOutputStream(samplerate=sample_rate, channels=1, dtype="int16", diff --git a/bolt_assistant/config.py b/bolt_assistant/config.py index ec588d3..df44f22 100644 --- a/bolt_assistant/config.py +++ b/bolt_assistant/config.py @@ -68,6 +68,10 @@ WAKE_WORDS = [w.strip().lower() for w in os.environ.get( "hey assistant,ok assistant,okay assistant," "hey bolts,hey volt,hey bold,hey boat,okay volt,hey thunder bolt,thunder bolt", ).split(",") if w.strip()] +# These count anywhere in what was said, not just at the start ("what's the +# weather, assistant?"), and the whole sentence is sent as the request. +ANYWHERE_WAKE_WORDS = [w.strip().lower() for w in os.environ.get( + "ANYWHERE_WAKE_WORDS", "assistant").split(",") if w.strip()] # ── listening (local voice-activity detection, before anything is sent) ──── # Nothing leaves the machine until this says someone is talking: silence and diff --git a/bolt_assistant/wake.py b/bolt_assistant/wake.py index da4ab31..737b1d0 100644 --- a/bolt_assistant/wake.py +++ b/bolt_assistant/wake.py @@ -12,6 +12,9 @@ Rules, all aimed at not waking up on ordinary conversation: that bolt" and "the bolt on the door" never count. - Whole words only: "boltage" is not "bolt". - Longest phrase wins, so "hey thunderbolt" isn't read as "hey" + garbage. +- Except *anywhere* words ("assistant"): one of those anywhere in what was + said counts, and the whole transcript is the request ("set a timer, + assistant" asks "set a timer, assistant"). """ from __future__ import annotations @@ -30,10 +33,11 @@ def _words(text: str) -> list: return [(m.group(0), m.start(), m.end()) for m in _WORD.finditer(text.lower())] -def find_wake(transcript: str, phrases: Iterable[str]) -> Tuple[bool, str]: - """(heard, remainder): whether a wake phrase opens *transcript*, and the - request that followed it in the same breath ("" if it was just the - phrase).""" +def find_wake(transcript: str, phrases: Iterable[str], + anywhere: Iterable[str] = ()) -> Tuple[bool, str]: + """(heard, remainder): whether a wake phrase opens *transcript* (or an + *anywhere* word appears in it), and the request that came with it in the + same breath ("" if it was just the phrase).""" words = _words(transcript or "") if not words: return False, "" @@ -47,4 +51,15 @@ def find_wake(transcript: str, phrases: Iterable[str]) -> Tuple[bool, str]: if len(span) == len(phrase) and all(w == p for (w, _, _), p in zip(span, phrase)): rest = transcript[span[-1][2]:] return True, rest.lstrip(" \t,.;:!?-—").strip() + return _find_anywhere(transcript, words, anywhere) + + +def _find_anywhere(transcript: str, words: list, anywhere: Iterable[str]) -> Tuple[bool, str]: + said = [w for w, _, _ in words] + for phrase in {tuple(_WORD.findall(p.lower())) for p in anywhere if p.strip()}: + n = len(phrase) + if any(tuple(said[i:i + n]) == phrase for i in range(len(said) - n + 1)): + if all(w in phrase or w in _FILLER for w in said): + return True, "" # just the word on its own: wait for the request + return True, transcript.strip() return False, "" diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 diff --git a/tests/test_assistant.py b/tests/test_assistant.py index 71d0498..19e4b8f 100644 --- a/tests/test_assistant.py +++ b/tests/test_assistant.py @@ -44,7 +44,6 @@ W = config.WAKE_WORDS ("tighten that bolt", (False, "")), ("the bolt on the door", (False, "")), ("I was bold", (False, "")), # the mishearing alone isn't a wake word - ("assistant set a timer", (False, "")), # "assistant" needs the "hey" ("boltage is high", (False, "")), ("", (False, "")), ]) @@ -52,6 +51,20 @@ def test_find_wake(said, expected): assert find_wake(said, W) == expected +@pytest.mark.parametrize("said, expected", [ + ("assistant set a timer", (True, "assistant set a timer")), + ("What's the weather, assistant?", (True, "What's the weather, assistant?")), + ("can the assistant turn it up", (True, "can the assistant turn it up")), + ("Hey assistant. Set a timer", (True, "Set a timer")), # at the start: stripped as before + ("Assistant.", (True, "")), # on its own: wait for the request + ("um, assistant", (True, "")), + ("my assistants are busy", (False, "")), # whole words only + ("tighten that bolt", (False, "")), +]) +def test_find_wake_anywhere(said, expected): + assert find_wake(said, W, config.ANYWHERE_WAKE_WORDS) == expected + + # ── voice-activity detection ──────────────────────────────────────────────── def seg(**kw):