themajesticmagicianandClaude Fable 5
ca65670638
stt.finish() used to run inline in the frame loop, deafening the assistant for up to ~4s per utterance while the server returned the transcript (and whatever was said meanwhile was later drained). The finish now runs on a background thread per utterance; the segmenter keeps consuming mic frames, so speech during the wait becomes its own utterance and is transcribed too. Transcripts are delivered oldest- first between frames, deferred while an utterance is in progress, and the command/follow-up window no longer times out on transcription latency. Speaking stays half-duplex (echo protection). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
398 lines
14 KiB
Python
398 lines
14 KiB
Python
"""Bolt Assistant without a mic, speaker or server: synthetic audio frames,
|
|
a fake transcriber, fake converse/tts/playback. Covers wake matching, the
|
|
voice-activity detector, the listen -> ask -> speak flow, and the command
|
|
rules."""
|
|
|
|
import array
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import pytest
|
|
|
|
from bolt_assistant import config, server
|
|
from bolt_assistant.assistant import Assistant, annotate
|
|
from bolt_assistant.listener import Segmenter
|
|
from bolt_assistant.stt import Transcription, parse_result
|
|
from bolt_assistant.tools import Tools
|
|
from bolt_assistant.wake import find_wake
|
|
|
|
LOUD, QUIET = 3000, 20
|
|
|
|
|
|
def frame(level, samples=1280):
|
|
return array.array("h", [level, -level] * (samples // 2)).tobytes()
|
|
|
|
|
|
# ── wake phrases ────────────────────────────────────────────────────────────
|
|
|
|
W = config.WAKE_WORDS
|
|
|
|
|
|
@pytest.mark.parametrize("said, expected", [
|
|
("Hey Bolt, what's the weather?", (True, "what's the weather?")),
|
|
("Bolt.", (True, "")),
|
|
("Thunderbolt turn on the lights", (True, "turn on the lights")),
|
|
("hey thunderbolt!", (True, "")),
|
|
("um, hey Bolt what time is it", (True, "what time is it")),
|
|
("Hey assistant. Set a timer", (True, "Set a timer")),
|
|
("Hey volt, play music", (True, "play music")), # a common mishearing, after "hey"
|
|
("okay so bolt, hi", (True, "hi")),
|
|
("tighten that bolt", (False, "")),
|
|
("the bolt on the door", (False, "")),
|
|
("I was bold", (False, "")), # the mishearing alone isn't a wake word
|
|
("boltage is high", (False, "")),
|
|
("", (False, "")),
|
|
])
|
|
def test_find_wake(said, expected):
|
|
assert find_wake(said, W) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("said, expected", [
|
|
("assistant set a timer", (True, "assistant set a timer")),
|
|
("What's the weather, assistant?", (True, "What's the weather, assistant?")),
|
|
("can the assistant turn it up", (True, "can the assistant turn it up")),
|
|
("Hey assistant. Set a timer", (True, "Set a timer")), # at the start: stripped as before
|
|
("Assistant.", (True, "")), # on its own: wait for the request
|
|
("um, assistant", (True, "")),
|
|
("my assistants are busy", (False, "")), # whole words only
|
|
("tighten that bolt", (False, "")),
|
|
])
|
|
def test_find_wake_anywhere(said, expected):
|
|
assert find_wake(said, W, config.ANYWHERE_WAKE_WORDS) == expected
|
|
|
|
|
|
# ── voice-activity detection ────────────────────────────────────────────────
|
|
|
|
def seg(**kw):
|
|
args = dict(threshold=500, start_frames=3, silence_end_sec=0.4, min_sec=0.3, max_sec=2.0)
|
|
args.update(kw)
|
|
return Segmenter(**args)
|
|
|
|
|
|
def feed(s, frames):
|
|
events = []
|
|
for f in frames:
|
|
events += s.feed(f)
|
|
return events
|
|
|
|
|
|
def test_silence_never_starts_an_utterance():
|
|
assert feed(seg(), [frame(QUIET)] * 50) == []
|
|
|
|
|
|
def test_speech_starts_with_pre_roll_and_ends_after_silence():
|
|
s = seg()
|
|
events = feed(s, [frame(QUIET)] * 6 + [frame(LOUD)] * 6 + [frame(QUIET)] * 5)
|
|
kinds = [e.kind for e in events]
|
|
assert kinds[0] == "start" and kinds[-1] == "end"
|
|
start = events[0]
|
|
assert len(start.frames) == 4 + 3 # pre-roll kept the first syllable, plus the trigger
|
|
end = events[-1]
|
|
assert len(end.audio) == (4 + 6 + 5) * 2560 # pre-roll + speech + trailing silence
|
|
|
|
|
|
def test_a_click_is_not_speech_and_a_short_burst_is_dropped():
|
|
s = seg(min_sec=0.5) # needs 6+ loud frames
|
|
assert feed(s, [frame(LOUD), frame(QUIET)] * 5) == [] # isolated clicks
|
|
events = feed(s, [frame(LOUD)] * 3 + [frame(QUIET)] * 5)
|
|
assert [e.kind for e in events if e.kind != "frame"] == ["start", "dropped"]
|
|
|
|
|
|
def test_long_speech_is_cut_off_at_the_cap():
|
|
events = feed(seg(max_sec=1.0), [frame(LOUD)] * 30)
|
|
ends = [e for e in events if e.kind == "end"]
|
|
assert ends and ends[0].seconds <= 1.0 + 1e-6
|
|
|
|
|
|
# ── the flow ────────────────────────────────────────────────────────────────
|
|
|
|
class FakeTranscription:
|
|
"""Each utterance takes the next scripted transcript. Bound at creation
|
|
(the "start" event, on the main thread) because finish() now runs on a
|
|
worker thread per utterance and two may be in flight at once."""
|
|
|
|
script = []
|
|
|
|
def __init__(self):
|
|
self.frames = 0
|
|
self.cancelled = False
|
|
self.transcript = FakeTranscription.script.pop(0)
|
|
|
|
def add(self, frames):
|
|
self.frames += len(frames)
|
|
|
|
def finish(self, audio):
|
|
return self.transcript
|
|
|
|
def cancel(self):
|
|
self.cancelled = True
|
|
|
|
|
|
class Clock:
|
|
def __init__(self):
|
|
self.t = 1000.0
|
|
|
|
def __call__(self):
|
|
return self.t
|
|
|
|
|
|
def utterance():
|
|
return [frame(LOUD)] * 8 + [frame(QUIET)] * 12
|
|
|
|
|
|
def make(replies, commands="read", streaming=False, transcription=FakeTranscription, **kw):
|
|
asked, played, logs = [], [], []
|
|
|
|
def converse(message, on_command, on_say=None):
|
|
asked.append(message)
|
|
reply = replies.pop(0)
|
|
if callable(reply):
|
|
return reply(on_command)
|
|
return server.Reply(reply)
|
|
|
|
clock = Clock()
|
|
a = Assistant(transcription=transcription, converse=converse,
|
|
converse_stream=None if not streaming else kw.pop("stream"),
|
|
tts=lambda text, voice: [text.encode()], play=lambda chunks: played.extend(chunks),
|
|
tools=Tools(mode=commands, log=logs.append), log=logs.append, clock=clock, **kw)
|
|
return a, asked, played, logs, clock
|
|
|
|
|
|
def say(a, transcript):
|
|
FakeTranscription.script.append(transcript)
|
|
for f in utterance():
|
|
a.process(f)
|
|
a.handle_transcripts(wait=True) # what run() does between frames
|
|
|
|
|
|
def spoken(played):
|
|
return [p.decode() for p in played if not p.startswith(b"\x00") and len(p) < 2000]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _config(monkeypatch):
|
|
monkeypatch.setattr(config, "STREAMING_REPLIES", False)
|
|
monkeypatch.setattr(config, "VAD_RMS_THRESHOLD", 500)
|
|
FakeTranscription.script = []
|
|
|
|
|
|
def test_ignores_speech_without_the_wake_phrase():
|
|
a, asked, played, logs, _ = make([])
|
|
say(a, "so anyway I was telling him about the bolt")
|
|
assert asked == [] and played == []
|
|
|
|
|
|
def test_wake_phrase_and_request_in_one_breath():
|
|
a, asked, played, logs, _ = make(["It's three o'clock."])
|
|
say(a, "Hey Bolt, what time is it?")
|
|
assert asked[0].startswith("what time is it?\n\n[Bolt Assistant: a voice-only assistant")
|
|
assert "It's three o'clock." in spoken(played)
|
|
assert a.state == "idle"
|
|
|
|
|
|
def test_bare_wake_phrase_chimes_then_takes_the_next_utterance():
|
|
a, asked, played, logs, clock = make(["Sure."])
|
|
say(a, "Hey Bolt")
|
|
assert a.state == "command" and asked == []
|
|
assert played # the chime
|
|
clock.t += 1
|
|
say(a, "turn it up") # no wake phrase needed now
|
|
assert asked[0].startswith("turn it up")
|
|
|
|
|
|
def test_the_command_window_closes():
|
|
a, asked, played, logs, clock = make([])
|
|
say(a, "Thunderbolt")
|
|
clock.t += config.COMMAND_WAIT_SEC + 5
|
|
say(a, "so then we left")
|
|
assert asked == [] and a.state == "idle"
|
|
|
|
|
|
def test_a_question_keeps_listening_for_the_answer():
|
|
a, asked, played, logs, clock = make(["Which room, the kitchen or the bedroom?", "Done."])
|
|
say(a, "Hey Bolt, turn on the lights")
|
|
assert a.state == "follow_up"
|
|
clock.t += 2
|
|
say(a, "the kitchen")
|
|
assert asked[1].startswith("the kitchen") and a.state == "idle"
|
|
|
|
|
|
def test_mic_audio_during_speech_is_discarded():
|
|
a, asked, played, logs, clock = make(["Hello there."])
|
|
a.on_mic_frame(frame(LOUD)) # queued while... (simulating backlog)
|
|
say(a, "hey bolt hello")
|
|
assert a.frames.empty() # what the mic heard while Bolt talked is gone
|
|
assert not a.segmenter.in_speech
|
|
|
|
|
|
def test_listening_continues_while_a_transcript_is_in_flight():
|
|
import threading
|
|
|
|
release = threading.Event()
|
|
|
|
class Slow(FakeTranscription):
|
|
def finish(self, audio):
|
|
release.wait(timeout=5)
|
|
return self.transcript
|
|
|
|
a, asked, played, logs, _ = make(["One.", "Two."], transcription=Slow)
|
|
FakeTranscription.script += ["hey bolt first thing", "hey bolt second thing"]
|
|
for f in utterance() + utterance(): # the second said while the first transcribes
|
|
a.process(f)
|
|
assert asked == [] # both transcripts still at the "server"
|
|
release.set()
|
|
a.handle_transcripts(wait=True)
|
|
assert len(asked) == 2 # nothing said during the wait was lost
|
|
assert asked[0].startswith("first thing") and asked[1].startswith("second thing")
|
|
|
|
|
|
def test_streamed_sentences_are_spoken_as_they_arrive():
|
|
def stream(message, on_say, on_command):
|
|
on_say("First.")
|
|
on_say("Second.")
|
|
return server.Reply("First. Second.", streamed=True)
|
|
|
|
a, asked, played, logs, _ = make([], streaming=True, stream=stream)
|
|
a._stream = stream
|
|
say(a, "hey bolt tell me two things")
|
|
assert spoken(played)[-2:] == ["First.", "Second."]
|
|
|
|
|
|
def test_server_failure_chimes_and_goes_back_to_listening():
|
|
def boom(on_command):
|
|
raise server.ServerError("down")
|
|
|
|
a, asked, played, logs, _ = make([boom])
|
|
say(a, "hey bolt anything")
|
|
assert any("down" in line for line in logs) and a.state == "idle"
|
|
|
|
|
|
def test_markdown_is_not_read_aloud():
|
|
a, asked, played, logs, _ = make(["**Sure** — see https://example.com for `details`."])
|
|
say(a, "hey bolt help")
|
|
text = spoken(played)[-1]
|
|
assert "**" not in text and "https" not in text and "`" not in text
|
|
|
|
|
|
def test_the_note_describes_what_bolt_may_do():
|
|
assert "can't run shell commands" in annotate("x", "read")
|
|
assert "`command: <shell command>`" in annotate("x", "all")
|
|
assert "can't act on this computer" in annotate("x", "off")
|
|
|
|
|
|
# ── commands relayed by the server ──────────────────────────────────────────
|
|
|
|
def filectl(**payload):
|
|
return "filectl " + json.dumps(payload)
|
|
|
|
|
|
def test_read_mode_lists_files_but_refuses_shell_and_writes(tmp_path):
|
|
(tmp_path / "a.txt").write_text("hi")
|
|
ran = []
|
|
tools = Tools(mode="read", log=lambda _m: None, shell=lambda c: ran.append(c) or "[exit 0]\n")
|
|
assert "a.txt" in tools(filectl(op="list", path=str(tmp_path)))
|
|
assert "hi" in tools(filectl(op="read", path=str(tmp_path / "a.txt")))
|
|
assert "Not run" in tools("rm -rf /")
|
|
assert "Not run" in tools(filectl(op="write", path=str(tmp_path / "b.txt"), content="x"))
|
|
assert ran == [] and not (tmp_path / "b.txt").exists()
|
|
|
|
|
|
def test_all_mode_runs_commands_and_off_runs_nothing(tmp_path):
|
|
ran = []
|
|
shell = lambda c: ran.append(c) or "[exit 0]\n" # noqa: E731
|
|
assert Tools(mode="all", log=lambda _m: None, shell=shell)("command: echo hi") == "[exit 0]\n"
|
|
assert ran == ["echo hi"] # the stray "command:" label was stripped
|
|
assert "Not run" in Tools(mode="off", log=lambda _m: None, shell=shell)(filectl(op="list", path="."))
|
|
assert "isn't available" in Tools(mode="all", log=lambda _m: None)("petctl emote wave")
|
|
|
|
|
|
def test_unknown_mode_falls_back_to_read():
|
|
assert Tools(mode="everything", log=lambda _m: None).mode == "read"
|
|
|
|
|
|
def test_a_relayed_command_goes_through_the_rules(tmp_path):
|
|
(tmp_path / "notes.txt").write_text("buy milk")
|
|
|
|
def reply_with_tool(on_command):
|
|
output = on_command(filectl(op="read", path=str(tmp_path / "notes.txt")))
|
|
assert "buy milk" in output
|
|
return server.Reply("Your note says buy milk.")
|
|
|
|
a, asked, played, logs, _ = make([reply_with_tool])
|
|
say(a, "hey bolt read my notes")
|
|
assert "Your note says buy milk." in spoken(played)
|
|
|
|
|
|
# ── server pieces ───────────────────────────────────────────────────────────
|
|
|
|
def test_whole_samples_carries_an_odd_byte_across_chunks():
|
|
out = b"".join(server.whole_samples([b"\x01", b"\x02\x03", b"\x04\x05"]))
|
|
assert out == b"\x01\x02\x03\x04" # the lone trailing byte waits for its partner
|
|
|
|
|
|
def test_unsaid_drops_what_was_already_spoken():
|
|
reply = server.unsaid(server.Reply("One. Two.\n\nThree."), ["One.", "Two."])
|
|
assert reply == server.Reply("Three.", streamed=False)
|
|
|
|
|
|
def test_tts_needs_a_voice(monkeypatch):
|
|
monkeypatch.setattr(config, "VOICE_ID", "")
|
|
with pytest.raises(server.TtsError, match="ELEVENLABS_VOICE_ID"):
|
|
server.tts_stream("hi")
|
|
|
|
|
|
def test_tts_request_shape(monkeypatch):
|
|
monkeypatch.setattr(config, "SERVER_URL", "http://s")
|
|
monkeypatch.setattr(config, "API_KEY", "k")
|
|
monkeypatch.setattr(config, "VOICE_ID", "v1")
|
|
response = MagicMock()
|
|
response.iter_content.return_value = [b"\x00\x01"]
|
|
with patch.object(server.requests, "post", return_value=response) as post:
|
|
assert list(server.tts_stream("hello", voice_id="v2")) == [b"\x00\x01"]
|
|
assert post.call_args.args[0] == "http://s/desk/tts"
|
|
assert post.call_args.kwargs["json"]["voice_id"] == "v2" # a speak_as pick wins
|
|
assert post.call_args.kwargs["headers"] == {"X-Desk-Api-Key": "k"}
|
|
|
|
|
|
def test_parse_result():
|
|
msg = json.dumps({"type": "Results", "is_final": True, "channel": {"alternatives": [{"transcript": "hey bolt"}]}})
|
|
assert parse_result(msg) == ("hey bolt", True)
|
|
assert parse_result("junk") == ("", False)
|
|
|
|
|
|
def test_transcription_falls_back_to_one_upload():
|
|
class Sock:
|
|
def __init__(self):
|
|
self.audio, self.replies = [], [json.dumps({"is_final": True, "channel": {
|
|
"alternatives": [{"transcript": "hey bolt"}]}}), ""]
|
|
|
|
def send_binary(self, data):
|
|
self.audio.append(data)
|
|
|
|
def send(self, _text):
|
|
pass
|
|
|
|
def recv(self):
|
|
return self.replies.pop(0)
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
late = Sock()
|
|
calls = iter([ConnectionError("down"), late])
|
|
|
|
def connect(_rate):
|
|
item = next(calls)
|
|
if isinstance(item, Exception):
|
|
raise item
|
|
return item
|
|
|
|
t = Transcription(connect)
|
|
t.add([frame(LOUD)]) # nowhere to go: the live socket never opened
|
|
assert t.finish(frame(LOUD) * 2) == "hey bolt"
|
|
assert b"".join(late.audio) == frame(LOUD) * 2
|