Streaming replies and STT, amplitude lip-sync, one place for speaking
Latency: replies are spoken sentence-by-sentence off the desk API's NDJSON endpoint, so the wait is time-to-first-sentence rather than the whole model call, and Deepgram's live websocket transcribes while you're still talking instead of uploading the WAV afterwards. Both fall back invisibly — a stream that fails before anything was said drops to converse(), and a socket that never opens just means the old one-shot path. Speaking lived in four near-copies in the controller (a reply, a holding line, a streamed sentence, a dialogue scene) that had already drifted: one didn't arm barge-in, another skipped the follow-up rule. It's now speech.Speaker plus an Utterance describing the policy differences, with collaborators injected so the whole of it tests without Qt or audio. The mouth follows the audio rather than a timer: tts.level_of reduces each PCM frame to a 0..1 loudness on a sqrt curve (speech sits well below peak, and a linear map leaves the mouth barely open during normal talking) and that indexes the talking frames, which the sprite script now draws as an openness ramp. Offline pyttsx3 has no waveform, so stale levels hand control back to the timed loop instead of freezing the mouth mid-syllable. Also: the pet starts where you left it (ignoring positions on monitors that are no longer connected, since restoring those faithfully is how it ends up somewhere unreachable), and `python -m bolt_pet --doctor` is a preflight that says what to do about each problem rather than only what's wrong. tests/test_pipeline_smoke.py breaks the pure-logic rule on purpose. Every unit test passed all week while notifications sat unspoken for minutes, the pet said things twice and [laughing] got read aloud — each an interaction between two individually-correct units. It drives whole turns against a real HTTP server on a loopback port, faking only the mic and the speakers. It found a NameError in the paint path that would have fired on every repaint while talking. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -22,6 +22,14 @@ def no_screen_probes(monkeypatch):
|
||||
monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_streaming(monkeypatch):
|
||||
"""Most tests drive the plain request/response path; leaving streaming on
|
||||
would have them attempt a real HTTP call, fail, and fall back — passing,
|
||||
slowly, for the wrong reason. The streaming path has its own tests."""
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ctrl():
|
||||
return controller_mod.PetController()
|
||||
@@ -41,10 +49,10 @@ def test_full_turn_happy_path(monkeypatch, ctrl):
|
||||
|
||||
monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's the weather")
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None: controller_mod.server_client.Reply("sunny and 72"))
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse", lambda text, on_command=None, on_say=None: controller_mod.server_client.Reply("sunny and 72"))
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1])
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1])
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
@@ -59,7 +67,7 @@ def test_turn_with_nothing_heard_returns_to_idle_without_calling_server(monkeypa
|
||||
monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: None)
|
||||
called = {"n": 0}
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: called.__setitem__("n", called["n"] + 1))
|
||||
lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1))
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
@@ -97,7 +105,7 @@ def test_turn_with_server_error_flashes_error_then_idle(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.mic, "record_utterance", lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "hello")
|
||||
|
||||
def boom(text, on_command=None):
|
||||
def boom(text, on_command=None, on_say=None):
|
||||
raise controller_mod.server_client.ServerError("server is down")
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse", boom)
|
||||
|
||||
@@ -150,7 +158,7 @@ def test_heartbeat_speaks_a_pending_announcement_when_idle(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: "don't forget your 3pm")
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: (spoken.append(text), True)[1])
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: (spoken.append(text), True)[1])
|
||||
said = _capture(ctrl.said)
|
||||
|
||||
ctrl._maybe_heartbeat()
|
||||
|
||||
@@ -29,6 +29,14 @@ def no_screen_probes(monkeypatch):
|
||||
monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_streaming(monkeypatch):
|
||||
"""Most tests drive the plain request/response path; leaving streaming on
|
||||
would have them attempt a real HTTP call, fail, and fall back — passing,
|
||||
slowly, for the wrong reason. The streaming path has its own tests."""
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ctrl():
|
||||
return controller_mod.PetController()
|
||||
@@ -160,7 +168,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch
|
||||
ctrl._barge_in = detector
|
||||
logs = _capture(ctrl.log)
|
||||
|
||||
def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None):
|
||||
def interrupted_playback(text, on_error=None, should_stop=None, voice_id=None, on_level=None):
|
||||
# What really happens: frames get scored during playback, then one
|
||||
# clears the threshold and playback aborts.
|
||||
detector._frames, detector._peak, detector._last = 7, 0.81, 0.81
|
||||
@@ -180,7 +188,7 @@ def test_the_interrupt_log_reports_what_fired_not_the_reset_counters(monkeypatch
|
||||
def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl):
|
||||
logs = _capture(ctrl.log)
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: False) # interrupted
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False) # interrupted
|
||||
|
||||
ctrl._speak("a very long explanation")
|
||||
|
||||
@@ -190,7 +198,7 @@ def test_interrupted_playback_queues_an_immediate_next_turn(monkeypatch, ctrl):
|
||||
|
||||
def test_uninterrupted_playback_does_not_queue_a_turn(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
ctrl._speak("short answer")
|
||||
assert not ctrl._talk_now.is_set()
|
||||
|
||||
@@ -202,7 +210,7 @@ def spoke(monkeypatch):
|
||||
"""Playback that always completes, so only the follow-up rule decides
|
||||
whether another turn is queued."""
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
|
||||
|
||||
def test_a_reply_ending_in_a_question_keeps_listening(spoke, ctrl):
|
||||
@@ -290,7 +298,7 @@ def test_follow_up_can_be_turned_off(spoke, monkeypatch, ctrl):
|
||||
|
||||
def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: False)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False)
|
||||
ctrl._follow_ups = 3
|
||||
|
||||
ctrl._speak("a very long explanation")
|
||||
@@ -301,7 +309,7 @@ def test_being_interrupted_restarts_the_chain(monkeypatch, ctrl):
|
||||
|
||||
def test_speech_is_recorded_in_the_history(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
ctrl._speak("**bold** reply")
|
||||
assert ctrl.history.last().text == "**bold** reply" # raw, for copy/paste
|
||||
|
||||
@@ -315,10 +323,10 @@ def test_the_active_window_rides_along_with_the_utterance(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.screen_context, "context_for",
|
||||
lambda text: f"{text}\n\n[on screen right now: app.py]")
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
sent = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: sent.append(text) or Reply("that's a KeyError"))
|
||||
lambda text, on_command=None, on_say=None: sent.append(text) or Reply("that's a KeyError"))
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
@@ -371,10 +379,10 @@ def test_napping_still_answers_when_spoken_to(monkeypatch, ctrl):
|
||||
lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "you awake?")
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: Reply("always"))
|
||||
lambda text, on_command=None, on_say=None: Reply("always"))
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True)
|
||||
ctrl.set_napping(True)
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
@@ -389,9 +397,9 @@ def test_notifications_are_forwarded_and_spoken(monkeypatch, ctrl):
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
sent, spoken = [], []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: sent.append(text) or Reply("your build is green"))
|
||||
lambda text, on_command=None, on_say=None: sent.append(text) or Reply("your build is green"))
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: spoken.append(text) or True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: spoken.append(text) or True)
|
||||
|
||||
ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
|
||||
ctrl._drain_notifications()
|
||||
@@ -411,7 +419,7 @@ def test_notifications_are_not_forwarded_while_napping(monkeypatch, ctrl):
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
called = {"n": 0}
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: called.__setitem__("n", called["n"] + 1))
|
||||
lambda text, on_command=None, on_say=None: called.__setitem__("n", called["n"] + 1))
|
||||
ctrl.set_napping(True)
|
||||
|
||||
ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
|
||||
@@ -507,9 +515,9 @@ def test_check_deliveries_runs_after_a_conversation_turn(monkeypatch, ctrl, tmp_
|
||||
lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "send me that file")
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: Reply("it's on the way"))
|
||||
lambda text, on_command=None, on_say=None: Reply("it's on the way"))
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None: True)
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
monkeypatch.setattr(controller_mod.config, "DELIVERED_FILES_DIR", tmp_path)
|
||||
monkeypatch.setattr(controller_mod.server_client, "list_outbox_files",
|
||||
lambda: [{"id": "abc", "name": "notes.txt", "size": 2}])
|
||||
@@ -531,10 +539,10 @@ def _voice_turn(monkeypatch, ctrl, reply, said="talk like a pirate"):
|
||||
lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: said)
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: reply)
|
||||
lambda text, on_command=None, on_say=None: reply)
|
||||
monkeypatch.setattr(
|
||||
controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None:
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
voices.append(voice_id) or True,
|
||||
)
|
||||
ctrl._handle_conversation_turn()
|
||||
@@ -625,7 +633,7 @@ def test_dialoguectl_never_reaches_the_shell(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
|
||||
lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm",
|
||||
lambda pcm, rate, should_stop=None: True)
|
||||
lambda pcm, rate, should_stop=None, on_level=None: True)
|
||||
|
||||
output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "[cheerfully] hi"}))
|
||||
|
||||
@@ -638,7 +646,7 @@ def test_a_scene_shows_in_the_bubble_with_the_delivery_tags_stripped(monkeypatch
|
||||
monkeypatch.setattr(controller_mod.config, "DIALOGUE_VOICES", "narrator:9BWtsMINqrJLrRacOk9x")
|
||||
monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
|
||||
lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
|
||||
said = _capture(ctrl.said)
|
||||
|
||||
ctrl._handle_command(_dialogue_command(
|
||||
@@ -656,7 +664,7 @@ def test_a_mid_turn_scene_returns_to_thinking_not_idle(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "ELEVENLABS_VOICE_ID", "aaorr6ZHIL88gEexu7dC")
|
||||
monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
|
||||
lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
|
||||
ctrl._state.transition(PetState.LISTENING)
|
||||
ctrl._state.transition(PetState.THINKING)
|
||||
states = _capture(ctrl.state_changed)
|
||||
@@ -676,7 +684,7 @@ def test_the_scene_uses_a_voice_the_server_picked_with_speak_as(monkeypatch, ctr
|
||||
return np.zeros(4, dtype=np.int16), 24000
|
||||
|
||||
monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue", capture)
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None: True)
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm", lambda pcm, rate, should_stop=None, on_level=None: True)
|
||||
ctrl._apply_voice(controller_mod.server_client.Reply("ok", "PICKEDvoice123456789", "Terence"))
|
||||
|
||||
ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"}))
|
||||
@@ -718,7 +726,7 @@ def test_talking_over_a_scene_is_reported_up_the_relay(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "synthesize_dialogue",
|
||||
lambda inputs, model_id=None, stability=None: (np.zeros(4, dtype=np.int16), 24000))
|
||||
monkeypatch.setattr(controller_mod.tts, "play_pcm",
|
||||
lambda pcm, rate, should_stop=None: False) # barge-in
|
||||
lambda pcm, rate, should_stop=None, on_level=None: False) # barge-in
|
||||
|
||||
output = ctrl._handle_command(_dialogue_command({"voice": "self", "text": "hi"}))
|
||||
|
||||
@@ -788,9 +796,9 @@ def test_coming_back_up_reports_to_the_server_and_speaks_the_reply(monkeypatch,
|
||||
path=state, now=1000.0)
|
||||
sent, spoken = [], []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: sent.append(text) or Reply("Good, it's up."))
|
||||
lambda text, on_command=None, on_say=None: sent.append(text) or Reply("Good, it's up."))
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None:
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
spoken.append(text) or True)
|
||||
|
||||
ctrl._report_self_restart()
|
||||
@@ -805,8 +813,230 @@ def test_an_ordinary_start_reports_nothing(monkeypatch, ctrl, tmp_path):
|
||||
monkeypatch.setattr(controller_mod.self_restart, "DEFAULT_STATE_PATH", tmp_path / "none.json")
|
||||
called = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None: called.append(text))
|
||||
lambda text, on_command=None, on_say=None: called.append(text))
|
||||
|
||||
ctrl._report_self_restart()
|
||||
|
||||
assert called == []
|
||||
|
||||
|
||||
# ── holding line: "give me a sec" while a tool runs ────────────────────────
|
||||
# The server used to discard whatever the model wrote alongside a tool call,
|
||||
# so the whole round trip was silence — and the model, with no evidence its
|
||||
# sentence landed, said it again in the final reply.
|
||||
|
||||
def test_a_holding_line_is_spoken_before_the_tool_runs(monkeypatch, ctrl):
|
||||
order = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
order.append(("spoke", text)) or True)
|
||||
monkeypatch.setattr(controller_mod.server_client, "run_local_command",
|
||||
lambda cmd: order.append(("ran", cmd)) or "[exit 0]")
|
||||
|
||||
ctrl._speak_holding("Give me a sec.")
|
||||
ctrl._handle_command("df -h /")
|
||||
|
||||
assert order == [("spoke", "Give me a sec."), ("ran", "df -h /")]
|
||||
|
||||
|
||||
def test_the_holding_line_shows_in_the_bubble_but_not_the_transcript(monkeypatch, ctrl):
|
||||
"""It's filler. The transcript should keep the actual answer."""
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
said = _capture(ctrl.said)
|
||||
|
||||
ctrl._speak_holding("I'll check on that — one moment.")
|
||||
|
||||
assert said == ["I'll check on that — one moment."]
|
||||
assert ctrl.history.entries() == []
|
||||
|
||||
|
||||
def test_a_holding_line_returns_to_waiting_not_idle(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
ctrl._state.transition(PetState.LISTENING)
|
||||
ctrl._state.transition(PetState.THINKING)
|
||||
states = _capture(ctrl.state_changed)
|
||||
|
||||
ctrl._speak_holding("one sec")
|
||||
|
||||
assert states == ["talking", "thinking"]
|
||||
|
||||
|
||||
def test_the_relay_speaks_whatever_the_server_attaches(monkeypatch, ctrl):
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
spoken.append(text) or True)
|
||||
monkeypatch.setattr(controller_mod.mic, "record_utterance",
|
||||
lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's in that folder?")
|
||||
|
||||
def fake_converse(text, on_command=None, on_say=None):
|
||||
on_say("Let me check that for you.") # what the server now sends
|
||||
on_command("filectl {\"op\": \"list\", \"path\": \"/tmp\"}")
|
||||
return Reply("It's got three files in it.")
|
||||
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse", fake_converse)
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["Let me check that for you.", "It's got three files in it."]
|
||||
|
||||
|
||||
# ── streamed replies ───────────────────────────────────────────────────────
|
||||
# Without streaming the pet waits out the entire model call before saying a
|
||||
# word. With it, the wait is time-to-first-sentence.
|
||||
|
||||
def test_each_sentence_is_spoken_as_it_arrives(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
spoken.append(text) or True)
|
||||
|
||||
def fake_stream(text, on_say, on_command=None, timeout=180.0):
|
||||
on_say("The disk is fine.")
|
||||
on_say("About sixty percent used.")
|
||||
return controller_mod.server_client.Reply(
|
||||
"The disk is fine. About sixty percent used.", spoken=True)
|
||||
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse_stream", fake_stream)
|
||||
monkeypatch.setattr(controller_mod.mic, "record_utterance",
|
||||
lambda *a, **k: np.zeros(10, dtype=np.int16))
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "how's the disk?")
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["The disk is fine.", "About sixty percent used."]
|
||||
# ...and the empty final reply is not spoken as an extra blank utterance.
|
||||
assert ctrl.history.entries()[-1].text == "About sixty percent used."
|
||||
|
||||
|
||||
def test_a_stream_that_fails_before_speaking_falls_back_silently(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
|
||||
spoken = []
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None:
|
||||
spoken.append(text) or True)
|
||||
|
||||
def broken_stream(text, on_say, on_command=None, timeout=180.0):
|
||||
raise controller_mod.server_client.ServerError("no streaming endpoint")
|
||||
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse_stream", broken_stream)
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None, on_say=None: Reply("Fell back fine."))
|
||||
|
||||
assert ctrl._ask_server("hello").text == "Fell back fine."
|
||||
assert spoken == [] # nothing was said twice
|
||||
|
||||
|
||||
def test_streaming_can_be_switched_off(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
|
||||
called = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse_stream",
|
||||
lambda *a, **k: called.append(1))
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None, on_say=None: Reply("plain path"))
|
||||
|
||||
assert ctrl._ask_server("hi").text == "plain path"
|
||||
assert called == []
|
||||
|
||||
|
||||
def test_talking_over_a_streamed_reply_still_interrupts(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: False)
|
||||
|
||||
ctrl._speak_stream_chunk("A long explanation you cut short.")
|
||||
|
||||
assert ctrl._talk_now.is_set()
|
||||
|
||||
|
||||
# ── notification latency (reported 2026-08-02: 5-10 minutes) ───────────────
|
||||
# Draining was wired to the heartbeat's 60s interval, and a heartbeat that
|
||||
# landed mid-conversation stamped its clock before noticing — so it burned the
|
||||
# slot and waited another full interval. Several of those in a row is minutes.
|
||||
|
||||
def test_a_notification_goes_out_on_the_next_tick_not_the_next_heartbeat(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "NOTIFICATION_MIN_INTERVAL_SECONDS", 0)
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
sent = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None, on_say=None:
|
||||
sent.append(text) or Reply("noted"))
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None)
|
||||
# A heartbeat has just run, so the next one is a full interval away.
|
||||
ctrl._last_heartbeat = controller_mod.time.monotonic()
|
||||
|
||||
ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
|
||||
ctrl._maybe_heartbeat()
|
||||
|
||||
assert sent and "Harry" in sent[0]
|
||||
|
||||
|
||||
def test_a_heartbeat_skipped_mid_conversation_does_not_burn_its_slot(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.server_client, "report_status", lambda: None)
|
||||
ctrl._last_heartbeat = 0.0
|
||||
ctrl._state.transition(PetState.LISTENING)
|
||||
ctrl._state.transition(PetState.THINKING)
|
||||
|
||||
ctrl._maybe_heartbeat() # due, but the pet is busy
|
||||
assert ctrl._last_heartbeat == 0.0, "the clock must not advance on a skipped tick"
|
||||
|
||||
ctrl._state.transition(PetState.IDLE)
|
||||
ctrl._maybe_heartbeat() # free now — runs immediately
|
||||
assert ctrl._last_heartbeat > 0.0
|
||||
|
||||
|
||||
def test_notifications_wait_while_the_pet_is_mid_turn(monkeypatch, ctrl):
|
||||
"""Speaking over the answer they're already getting would be worse than
|
||||
waiting a couple of seconds."""
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
sent = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None, on_say=None: sent.append(text))
|
||||
ctrl._state.transition(PetState.LISTENING)
|
||||
|
||||
ctrl._queue_notification(Notification(app="CI", summary="Build finished", body=""))
|
||||
ctrl._maybe_heartbeat()
|
||||
|
||||
assert sent == []
|
||||
assert len(ctrl._pending_notifications) == 1 # kept, not dropped
|
||||
|
||||
|
||||
def test_a_second_message_inside_the_rate_limit_is_no_longer_lost(monkeypatch, ctrl):
|
||||
"""The gate used to drop it. With NOTIFICATION_MIN_INTERVAL_SECONDS=60,
|
||||
two texts a minute apart meant you heard about one of them."""
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 60)
|
||||
sent = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "converse",
|
||||
lambda text, on_command=None, on_say=None:
|
||||
sent.append(text) or Reply("ok"))
|
||||
monkeypatch.setattr(controller_mod.tts, "speak",
|
||||
lambda text, on_error=None, should_stop=None, voice_id=None, on_level=None: True)
|
||||
|
||||
ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
|
||||
ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="it's urgent"))
|
||||
ctrl._drain_notifications()
|
||||
|
||||
assert len(sent) == 1 # one round trip...
|
||||
assert "you there?" in sent[0] and "it's urgent" in sent[0] # ...both messages
|
||||
assert "2 desktop notifications" in sent[0]
|
||||
|
||||
|
||||
def test_the_filter_still_applies(ctrl):
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("deploy", 0)
|
||||
ctrl._queue_notification(Notification(app="Chat", summary="lunch?", body=""))
|
||||
assert ctrl._pending_notifications == []
|
||||
|
||||
|
||||
def test_a_notification_storm_cannot_become_an_unbounded_backlog(ctrl):
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
for index in range(40):
|
||||
ctrl._queue_notification(Notification(app="Spam", summary=f"#{index}", body=""))
|
||||
|
||||
assert len(ctrl._pending_notifications) == controller_mod._MAX_PENDING_NOTIFICATIONS
|
||||
assert "#39" in ctrl._pending_notifications[-1].as_text() # newest kept
|
||||
|
||||
+12
-1
@@ -161,7 +161,18 @@ def test_the_relay_report_names_the_cast():
|
||||
action = dialogue.parse('dialoguectl ' + _scene(
|
||||
'{"voice": "self", "text": "one"}', '{"voice": "narrator", "text": "two"}',
|
||||
))
|
||||
assert dialogue.describe(action) == "[dialogue] played 2 lines in 2 voices: narrator, self"
|
||||
assert dialogue.describe(action).startswith(
|
||||
"[dialogue] played 2 lines in 2 voices: narrator, self")
|
||||
|
||||
|
||||
def test_the_report_says_the_scene_was_already_heard():
|
||||
"""Observed 2026-07-31: the scene played, then the final reply summarised
|
||||
it, so the pet said the same thing twice with nothing in between. The
|
||||
model cannot know the audio already happened unless it is told."""
|
||||
action = dialogue.parse('dialoguectl {"lines": [{"text": "hello"}]}')
|
||||
report = dialogue.describe(action)
|
||||
assert "HEARD this already" in report
|
||||
assert "Do not repeat" in report
|
||||
|
||||
|
||||
# ── the HTTP request ────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""The preflight. Its one job is to never be the thing that's broken.
|
||||
|
||||
A doctor that raises on a broken install diagnoses the wrong patient, so the
|
||||
tests that matter here are the ugly-input ones: no config at all, a check that
|
||||
throws, a dependency missing. The individual diagnoses are simple enough to
|
||||
read; that they *run* on a machine missing everything is the property worth
|
||||
pinning down.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet import config, doctor
|
||||
from bolt_pet.doctor import FAIL, OK, WARN
|
||||
|
||||
|
||||
def test_every_check_returns_a_verdict_on_a_bare_machine(monkeypatch):
|
||||
"""Nothing configured, nothing installed — still a full report."""
|
||||
for name in ("SERVER_URL", "API_KEY", "DEEPGRAM_API_KEY",
|
||||
"ELEVENLABS_API_KEY", "ELEVENLABS_VOICE_ID"):
|
||||
monkeypatch.setattr(config, name, "")
|
||||
monkeypatch.setattr(doctor, "_module", lambda _n: False)
|
||||
|
||||
results = doctor.run()
|
||||
|
||||
assert len(results) == len(doctor.CHECKS)
|
||||
assert all(c.status in (OK, WARN, FAIL) for c in results)
|
||||
assert all(c.name and c.detail for c in results)
|
||||
|
||||
|
||||
def test_a_check_that_raises_does_not_hide_the_others():
|
||||
"""One broken probe must not cost you the other eleven diagnoses."""
|
||||
def explode(*_args):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
original = doctor.CHECKS
|
||||
doctor.CHECKS = (("mic", explode),) + original[:2]
|
||||
try:
|
||||
results = doctor.run()
|
||||
finally:
|
||||
doctor.CHECKS = original
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0].status == FAIL
|
||||
assert "boom" in results[0].detail
|
||||
|
||||
|
||||
def test_missing_server_config_is_a_failure_not_a_warning(monkeypatch):
|
||||
"""Without it the controller exits its thread at startup — the pet looks
|
||||
alive and simply never answers. That is the worst failure mode there is."""
|
||||
monkeypatch.setattr(config, "missing_config", lambda: ["BOLT_SERVER_URL", "DESK_API_KEY"])
|
||||
check = doctor.check_config()
|
||||
assert check.status == FAIL
|
||||
assert "BOLT_SERVER_URL" in check.detail
|
||||
assert check.fix
|
||||
|
||||
|
||||
def test_a_configured_server_passes_without_being_contacted(monkeypatch):
|
||||
"""The shallow run must not need the network — it's the first thing you
|
||||
reach for when the network is what's wrong."""
|
||||
monkeypatch.setattr(config, "missing_config", lambda: [])
|
||||
monkeypatch.setattr(config, "SERVER_URL", "http://bolt.local:8000")
|
||||
assert doctor.check_config().status == OK
|
||||
assert doctor.check_server(deep=False).status == OK
|
||||
|
||||
|
||||
def test_every_problem_comes_with_something_to_do_about_it(monkeypatch):
|
||||
""""screen reading: warn" is useless on its own; "apt install tesseract-ocr"
|
||||
is the entire point of the tool."""
|
||||
for name in ("SERVER_URL", "API_KEY", "DEEPGRAM_API_KEY"):
|
||||
monkeypatch.setattr(config, name, "")
|
||||
monkeypatch.setattr(doctor, "_module", lambda _n: False)
|
||||
|
||||
for check in doctor.run():
|
||||
if check.status == FAIL:
|
||||
assert check.fix, f"{check.name} says what's wrong but not what to do"
|
||||
|
||||
|
||||
def test_the_exit_code_is_nonzero_only_for_real_failures(monkeypatch, capsys):
|
||||
monkeypatch.setattr(doctor, "run", lambda deep=False: [
|
||||
doctor.Check("a", OK, "fine"), doctor.Check("b", WARN, "degraded")])
|
||||
assert doctor.main([]) == 0
|
||||
|
||||
monkeypatch.setattr(doctor, "run", lambda deep=False: [doctor.Check("a", FAIL, "broken")])
|
||||
assert doctor.main([]) == 1
|
||||
assert "Fix those first" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_deep_is_off_unless_asked(monkeypatch):
|
||||
seen = []
|
||||
monkeypatch.setattr(doctor, "run", lambda deep=False: seen.append(deep) or [])
|
||||
doctor.main([])
|
||||
doctor.main(["--deep"])
|
||||
assert seen == [False, True]
|
||||
|
||||
|
||||
def test_a_slow_silence_timeout_is_flagged(monkeypatch):
|
||||
"""The setting most likely to make it feel sluggish, and the least obvious
|
||||
— it is pure dead air before anything at all starts happening."""
|
||||
monkeypatch.setattr(config, "SILENCE_END_SEC", 2.0)
|
||||
check = doctor.check_latency()
|
||||
assert check.status == WARN
|
||||
assert "2s" in check.detail or "2 " in check.detail
|
||||
|
||||
monkeypatch.setattr(config, "SILENCE_END_SEC", 0.9)
|
||||
assert doctor.check_latency().status == OK
|
||||
|
||||
|
||||
def test_a_check_line_renders_the_fix_only_when_there_is_a_problem():
|
||||
assert "→" not in doctor.Check("x", OK, "all good", fix="unused").line()
|
||||
assert "→ do the thing" in doctor.Check("x", WARN, "hmm", fix="do the thing").line()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Amplitude → mouth openness, from PCM samples to the drawn frame.
|
||||
|
||||
Two halves, tested separately because only one of them needs a display:
|
||||
`tts.level_of`/`envelope` turn samples into a 0..1 loudness, and
|
||||
`PetWindow._mouth_frame` turns that loudness into a frame index. They meet at
|
||||
the `mouth` signal, which the pipeline smoke test covers end to end.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet.audio import tts
|
||||
|
||||
|
||||
def frame(amplitude: int, samples: int = 480) -> np.ndarray:
|
||||
return np.full(samples, amplitude, dtype=np.int16)
|
||||
|
||||
|
||||
# ── loudness ────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_silence_closes_the_mouth():
|
||||
assert tts.level_of(np.zeros(480, dtype=np.int16)) == 0.0
|
||||
|
||||
|
||||
def test_a_loud_frame_opens_it_fully():
|
||||
assert tts.level_of(frame(30000)) == 1.0
|
||||
|
||||
|
||||
def test_the_level_never_leaves_zero_to_one():
|
||||
"""It indexes a frame list; out of range is an IndexError on the UI thread."""
|
||||
for amplitude in (0, 1, 500, 6000, 20000, 32767):
|
||||
assert 0.0 <= tts.level_of(frame(amplitude)) <= 1.0
|
||||
|
||||
|
||||
def test_it_rises_with_amplitude():
|
||||
quiet, middling, loud = (tts.level_of(frame(a)) for a in (800, 4000, 12000))
|
||||
assert quiet < middling < loud
|
||||
|
||||
|
||||
def test_quiet_speech_still_moves_the_mouth_visibly():
|
||||
"""The sqrt curve is the whole point. Speech spends most of its time well
|
||||
below peak, so a linear map leaves the mouth barely open for normal talking
|
||||
and the pet looks like it's mumbling."""
|
||||
assert tts.level_of(frame(1500)) > 0.15
|
||||
|
||||
|
||||
def test_an_empty_frame_is_silence_not_a_crash():
|
||||
assert tts.level_of(np.zeros(0, dtype=np.int16)) == 0.0
|
||||
|
||||
|
||||
# ── the envelope of a whole clip ────────────────────────────────────────────
|
||||
|
||||
def test_an_envelope_has_one_value_per_frame_at_the_requested_rate():
|
||||
one_second = np.zeros(16000, dtype=np.int16)
|
||||
assert len(tts.envelope(one_second, 16000, fps=30)) == pytest.approx(30, abs=1)
|
||||
|
||||
|
||||
def test_an_envelope_tracks_loud_and_quiet_stretches():
|
||||
pcm = np.concatenate([np.zeros(8000, dtype=np.int16), frame(20000, 8000)])
|
||||
levels = tts.envelope(pcm, 16000, fps=10)
|
||||
assert max(levels[:4]) == 0.0 # the silent half
|
||||
assert min(levels[-4:]) > 0.5 # the loud half
|
||||
|
||||
|
||||
def test_an_empty_clip_has_an_empty_envelope():
|
||||
assert tts.envelope(np.zeros(0, dtype=np.int16), 16000) == []
|
||||
|
||||
|
||||
# ── the drawn frame ─────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def window():
|
||||
"""Needs a QApplication; run with QT_QPA_PLATFORM=offscreen."""
|
||||
from PySide6.QtWidgets import QApplication
|
||||
|
||||
from bolt_pet.ui.pet_window import PetWindow
|
||||
|
||||
_app = QApplication.instance() or QApplication(["test"])
|
||||
win = PetWindow()
|
||||
yield win
|
||||
win.close()
|
||||
|
||||
|
||||
class FakeAnimation:
|
||||
"""Stands in for sprite.Animation — _mouth_frame only wants `.frames`."""
|
||||
|
||||
def __init__(self, frames):
|
||||
self.frames = list(frames)
|
||||
|
||||
|
||||
def test_the_talking_frame_follows_the_level(window):
|
||||
"""The talking frames are an openness ramp — closed first, widest last —
|
||||
specifically so loudness can index them directly."""
|
||||
animation = FakeAnimation(["closed", "a", "b", "c", "d", "open"])
|
||||
|
||||
assert window._mouth_frame(animation) is None # no level set yet
|
||||
|
||||
window.set_mouth(0.0)
|
||||
assert window._mouth_frame(animation) == "closed"
|
||||
|
||||
window.set_mouth(1.0)
|
||||
assert window._mouth_frame(animation) == "open"
|
||||
|
||||
window.set_mouth(0.5)
|
||||
assert window._mouth_frame(animation) not in ("closed", "open")
|
||||
|
||||
|
||||
def test_a_stale_level_gives_the_animation_back(window):
|
||||
"""If the audio thread stops sending levels — TTS died, the clip ended
|
||||
without a final zero — the mouth must not freeze half-open forever. After a
|
||||
moment it falls back to the ordinary looping animation."""
|
||||
animation = FakeAnimation(["a", "b", "c"])
|
||||
window.set_mouth(0.9)
|
||||
assert window._mouth_frame(animation) is not None
|
||||
|
||||
window._mouth_at = time.monotonic() - 5.0
|
||||
assert window._mouth_frame(animation) is None
|
||||
|
||||
|
||||
def test_a_single_frame_animation_falls_back_instead_of_indexing(window):
|
||||
"""Art with one talking frame can't lip-sync; it must not try."""
|
||||
window.set_mouth(1.0)
|
||||
assert window._mouth_frame(FakeAnimation(["only"])) is None
|
||||
assert window._mouth_frame(FakeAnimation([])) is None
|
||||
|
||||
|
||||
def test_no_animation_at_all_is_handled(window):
|
||||
window.set_mouth(0.5)
|
||||
assert window._mouth_frame(None) is None
|
||||
@@ -0,0 +1,289 @@
|
||||
"""End-to-end: microphone in, speech out, over real HTTP.
|
||||
|
||||
Every other test in this suite injects a fake at the seam it cares about, and
|
||||
every one of them passed all week while these got through to production:
|
||||
|
||||
- notifications sitting unspoken for minutes (a clock stamped in the wrong
|
||||
order, two correct units)
|
||||
- the pet saying the same thing twice (server discarded prose, model repeated
|
||||
it — both sides behaving as written)
|
||||
- `[laughing]` read out loud (a tag that means something to one model and
|
||||
nothing to the next one down the pipe)
|
||||
- a device command written as prose (extractor fine, prompt fine, no marker)
|
||||
|
||||
They were all *interaction* bugs. So this one runs the actual pipeline against
|
||||
a real socket: a threaded HTTP server that speaks the desk protocol, the real
|
||||
`server_client` doing real requests (including NDJSON streaming), the real
|
||||
controller loop and state machine. The only fakes are where the hardware is —
|
||||
the mic stream and the speakers — because those are the two things a test
|
||||
genuinely cannot have.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PySide6.QtWidgets import QApplication
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet import controller as controller_mod
|
||||
from bolt_pet.notifications import Notification
|
||||
from bolt_pet.state import PetState
|
||||
|
||||
_app = QApplication.instance() or QApplication(["test"])
|
||||
|
||||
|
||||
# ── a desk server that actually listens on a port ───────────────────────────
|
||||
|
||||
class FakeDesk:
|
||||
"""Scripted responses, real HTTP. Set `.script` per test."""
|
||||
|
||||
def __init__(self):
|
||||
self.script = {}
|
||||
self.requests = []
|
||||
self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler())
|
||||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
host, port = self._server.server_address[:2]
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def stop(self) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
|
||||
def _handler(self):
|
||||
desk = self
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_args):
|
||||
pass # the test output is not an access log
|
||||
|
||||
def do_GET(self):
|
||||
path = self.path.split("?")[0]
|
||||
desk.requests.append(("GET", path))
|
||||
self._json(desk.script.get(path, {"files": []}))
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
path = self.path.split("?")[0]
|
||||
desk.requests.append(("POST", path, body))
|
||||
response = desk.script.get(path)
|
||||
if callable(response):
|
||||
response = response(body)
|
||||
if path.endswith("converse_stream"):
|
||||
self._ndjson(response or [])
|
||||
else:
|
||||
self._json(response if response is not None else {"type": "reply", "text": "ok"})
|
||||
|
||||
def _json(self, payload):
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def _ndjson(self, events):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/x-ndjson")
|
||||
self.end_headers()
|
||||
for event in events:
|
||||
self.wfile.write((json.dumps(event) + "\n").encode())
|
||||
self.wfile.flush()
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
class FakeMic:
|
||||
"""Loud frames then quiet ones, so the VAD ends the utterance on its own."""
|
||||
|
||||
def __init__(self, loud=8, quiet=60):
|
||||
self.frames = ([np.full((320, 1), 4000, dtype=np.int16)] * loud
|
||||
+ [np.zeros((320, 1), dtype=np.int16)] * quiet)
|
||||
|
||||
def read(self, _n):
|
||||
return (self.frames.pop(0) if self.frames
|
||||
else np.zeros((320, 1), dtype=np.int16)), None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_a):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pipeline(monkeypatch):
|
||||
"""A controller wired to a real local server, with fake ears and mouth."""
|
||||
desk = FakeDesk()
|
||||
spoken: list[str] = []
|
||||
levels: list[float] = []
|
||||
|
||||
monkeypatch.setattr(controller_mod.config, "SERVER_URL", desk.url)
|
||||
monkeypatch.setattr(controller_mod.config, "API_KEY", "test-key")
|
||||
monkeypatch.setattr(controller_mod.config, "SESSION_ID", "pet-smoke")
|
||||
monkeypatch.setattr(controller_mod.config, "SILENCE_END_SEC", 0.2)
|
||||
monkeypatch.setattr(controller_mod.config, "MIN_UTTERANCE_S", 0.0)
|
||||
monkeypatch.setattr(controller_mod.config, "RECEIVE_FILES", False)
|
||||
# Off by default so each test picks its own path; the streaming tests opt in.
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", False)
|
||||
monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text)
|
||||
monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False)
|
||||
# Deepgram and the speakers are the two things a test can't have.
|
||||
monkeypatch.setattr(controller_mod.stt, "transcribe", lambda pcm: "what's on my disk?")
|
||||
monkeypatch.setattr(controller_mod.stt_stream.StreamingTranscriber, "open",
|
||||
classmethod(lambda cls, **kw: None))
|
||||
|
||||
def fake_speak(text, on_error=None, should_stop=None, voice_id=None, on_level=None):
|
||||
spoken.append(text)
|
||||
if on_level is not None:
|
||||
on_level(0.8) # the mouth opens while a word plays...
|
||||
levels.append(0.8)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(controller_mod.tts, "speak", fake_speak)
|
||||
|
||||
ctrl = controller_mod.PetController()
|
||||
ctrl._stream = FakeMic()
|
||||
try:
|
||||
yield ctrl, desk, spoken, levels
|
||||
finally:
|
||||
desk.stop()
|
||||
|
||||
|
||||
# ── the whole path ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_a_plain_turn_goes_mic_to_speaker(pipeline):
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
desk.script["/desk/converse"] = {"type": "reply", "text": "About sixty percent full."}
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["About sixty percent full."]
|
||||
assert ctrl._state.state == PetState.IDLE
|
||||
posted = [r for r in desk.requests if r[0] == "POST"]
|
||||
assert posted[0][1] == "/desk/converse"
|
||||
assert "what's on my disk?" in posted[0][2]["text"]
|
||||
assert ctrl.history.entries()[-1].text == "About sixty percent full."
|
||||
|
||||
|
||||
def test_a_streamed_turn_speaks_each_sentence_as_it_lands(pipeline, monkeypatch):
|
||||
"""The NDJSON is parsed by the real client over a real socket — the layer
|
||||
that a mocked `converse` can never exercise."""
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
|
||||
desk.script["/desk/converse_stream"] = [
|
||||
{"type": "say", "text": "The disk is fine."},
|
||||
{"type": "say", "text": "About sixty percent used."},
|
||||
{"type": "reply", "text": "The disk is fine. About sixty percent used.",
|
||||
"already_spoken": True},
|
||||
]
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["The disk is fine.", "About sixty percent used."]
|
||||
# The final reply must not be spoken a third time.
|
||||
assert len(spoken) == 2
|
||||
|
||||
|
||||
def test_a_tool_turn_says_give_me_a_sec_then_the_answer(pipeline, monkeypatch):
|
||||
"""Holding line, relayed command, and the real answer — in that order."""
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
ran = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "run_local_command",
|
||||
lambda cmd: ran.append(cmd) or "[exit 0]\n60% used")
|
||||
desk.script["/desk/converse"] = {
|
||||
"type": "command", "command": "df -h /", "token": "tok",
|
||||
"say": "Let me check that for you.",
|
||||
}
|
||||
desk.script["/desk/tool_result"] = {"type": "reply", "text": "Sixty percent used."}
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["Let me check that for you.", "Sixty percent used."]
|
||||
assert ran == ["df -h /"]
|
||||
relayed = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/tool_result")
|
||||
assert relayed[2]["output"].endswith("60% used")
|
||||
|
||||
|
||||
def test_a_device_command_never_reaches_the_shell(pipeline, monkeypatch):
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
ran = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "run_local_command", lambda cmd: ran.append(cmd))
|
||||
actions = []
|
||||
ctrl.action.connect(actions.append)
|
||||
desk.script["/desk/converse"] = {
|
||||
"type": "command", "command": "petctl emote wave", "token": "tok"}
|
||||
desk.script["/desk/tool_result"] = {"type": "reply", "text": "There you go."}
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert ran == []
|
||||
assert actions == [{"action": "emote", "emote": "wave"}]
|
||||
assert spoken == ["There you go."]
|
||||
|
||||
|
||||
def test_the_mouth_moves_with_the_audio(pipeline):
|
||||
"""Lip-sync is only real if the level actually reaches the window."""
|
||||
ctrl, desk, _spoken, levels = pipeline
|
||||
mouth = []
|
||||
ctrl.mouth.connect(mouth.append)
|
||||
desk.script["/desk/converse"] = {"type": "reply", "text": "Talking now."}
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert levels, "TTS was never given a level callback"
|
||||
assert 0.8 in mouth # opened while speaking...
|
||||
assert mouth[-1] == 0.0 # ...and closed at the end
|
||||
|
||||
|
||||
def test_a_notification_is_forwarded_and_spoken(pipeline):
|
||||
"""The path that was silently sitting for five to ten minutes."""
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
ctrl._notification_gate = controller_mod.notifications.NotificationGate("", 0)
|
||||
desk.script["/desk/converse"] = {"type": "reply", "text": "Harry says he's around."}
|
||||
|
||||
ctrl._queue_notification(Notification(app="Signal", summary="Harry", body="you there?"))
|
||||
ctrl._maybe_heartbeat()
|
||||
|
||||
assert spoken == ["Harry says he's around."]
|
||||
forwarded = next(r for r in desk.requests if r[0] == "POST" and r[1] == "/desk/converse")
|
||||
assert "Harry" in forwarded[2]["text"]
|
||||
|
||||
|
||||
def test_streaming_falls_back_when_the_server_is_older(pipeline, monkeypatch):
|
||||
"""A server without /desk/converse_stream (or one that answers with nothing)
|
||||
must not cost a turn — the client drops to the plain endpoint. This is how
|
||||
the pet keeps working against a container that hasn't been updated yet."""
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
monkeypatch.setattr(controller_mod.config, "STREAMING_REPLIES", True)
|
||||
desk.script["/desk/converse_stream"] = [] # nothing streamed back
|
||||
desk.script["/desk/converse"] = {"type": "reply", "text": "Still here."}
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert spoken == ["Still here."]
|
||||
paths = [r[1] for r in desk.requests if r[0] == "POST"]
|
||||
assert paths == ["/desk/converse_stream", "/desk/converse"]
|
||||
|
||||
|
||||
def test_a_dead_server_leaves_the_pet_usable(pipeline):
|
||||
"""It should flash an error and go back to listening, not wedge."""
|
||||
ctrl, desk, spoken, _levels = pipeline
|
||||
desk.stop() # the server disappears mid-session
|
||||
states = []
|
||||
ctrl.state_changed.connect(states.append)
|
||||
|
||||
ctrl._handle_conversation_turn()
|
||||
|
||||
assert states[-2:] == ["error", "idle"]
|
||||
assert spoken == []
|
||||
@@ -0,0 +1,260 @@
|
||||
"""The rules about talking — the four kinds of utterance and the mic policy.
|
||||
|
||||
These were four near-copies in the controller before, and the copies had
|
||||
drifted: one didn't arm barge-in, one skipped the follow-up rule. The value of
|
||||
having one `Speaker` is only real if the differences between the kinds stay
|
||||
*visible*, so this asserts on the differences rather than on the machinery.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet import config, speech
|
||||
from bolt_pet.speech import Speaker, Utterance
|
||||
from bolt_pet.state import PetState, PetStateMachine
|
||||
|
||||
|
||||
class FakeTts:
|
||||
def __init__(self, completed=True):
|
||||
self.completed = completed
|
||||
self.calls = []
|
||||
|
||||
def speak(self, text, on_error=None, should_stop=None, voice_id=None, on_level=None):
|
||||
self.calls.append({"text": text, "voice_id": voice_id,
|
||||
"interruptible": should_stop is not None})
|
||||
if on_level is not None:
|
||||
on_level(0.7)
|
||||
return self.completed
|
||||
|
||||
def play_pcm(self, pcm, sample_rate, should_stop=None, on_level=None):
|
||||
self.calls.append({"pcm": pcm, "rate": sample_rate,
|
||||
"interruptible": should_stop is not None})
|
||||
return self.completed
|
||||
|
||||
|
||||
class FakeBargeIn:
|
||||
def __init__(self):
|
||||
self.resets = 0
|
||||
|
||||
def reset(self):
|
||||
self.resets += 1
|
||||
|
||||
def check(self, _frame=None):
|
||||
return False
|
||||
|
||||
|
||||
def build(**kwargs):
|
||||
"""A speaker plus the things worth asserting on."""
|
||||
state = PetStateMachine()
|
||||
tts = kwargs.pop("tts", None) or FakeTts()
|
||||
said, logged, recorded, levels = [], [], [], []
|
||||
speaker = Speaker(
|
||||
state=state, tts=tts,
|
||||
history=recorded.append,
|
||||
on_said=said.append,
|
||||
on_log=logged.append,
|
||||
on_level=levels.append,
|
||||
**kwargs,
|
||||
)
|
||||
return speaker, state, tts, said, logged, recorded, levels
|
||||
|
||||
|
||||
# ── what distinguishes the four kinds ───────────────────────────────────────
|
||||
|
||||
def test_a_reply_is_recorded_but_a_holding_line_is_not():
|
||||
"""Filler must not push the actual answer out of the transcript."""
|
||||
speaker, state, _tts, _said, _logged, recorded, _levels = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.holding("Give me a sec."))
|
||||
speaker.say(Utterance.reply("Sixty percent used."))
|
||||
|
||||
assert recorded == ["Sixty percent used."]
|
||||
|
||||
|
||||
def test_a_holding_line_resumes_the_turn_it_interrupted():
|
||||
"""The turn isn't over — a tool is still running — so it must go back to
|
||||
THINKING, not drop to IDLE and end the turn."""
|
||||
speaker, state, _tts, *_ = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.holding("Let me check."))
|
||||
|
||||
assert state.state == PetState.THINKING
|
||||
|
||||
|
||||
def test_a_holding_line_cannot_be_talked_over():
|
||||
"""Cutting off "give me a sec" strands the tool that's already running."""
|
||||
detector = FakeBargeIn()
|
||||
speaker, state, tts, *_ = build(barge_in=lambda: detector)
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.holding("One sec."))
|
||||
assert tts.calls[-1]["interruptible"] is False
|
||||
|
||||
speaker.say(Utterance.reply("Done."))
|
||||
assert tts.calls[-1]["interruptible"] is True
|
||||
|
||||
|
||||
def test_a_streamed_sentence_stays_talking_between_sentences():
|
||||
"""Otherwise the sprite flickers idle-talking-idle down a long answer."""
|
||||
speaker, state, *_ = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.stream_chunk("The disk is fine."))
|
||||
|
||||
assert state.state == PetState.TALKING
|
||||
|
||||
|
||||
def test_a_scene_is_recorded_because_the_user_heard_it():
|
||||
speaker, state, _tts, _said, _logged, recorded, _levels = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say_pcm(Utterance.scene("Once upon a time."), pcm=b"\x00\x00", sample_rate=44100)
|
||||
|
||||
assert recorded == ["Once upon a time."]
|
||||
assert state.state == PetState.THINKING
|
||||
|
||||
|
||||
# ── barge-in ordering, which is what the copies got wrong ───────────────────
|
||||
|
||||
def test_the_detector_is_reset_after_playback_not_only_before():
|
||||
"""Playback fed the pet's own voice into the wake model's window. If it
|
||||
isn't cleared afterwards, the idle listener re-hears the last sentence and
|
||||
the pet answers itself."""
|
||||
detector = FakeBargeIn()
|
||||
speaker, state, *_ = build(barge_in=lambda: detector)
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("Hello there."))
|
||||
|
||||
assert detector.resets == 2 # armed before, cleared after
|
||||
|
||||
|
||||
def test_the_barge_in_detail_is_captured_before_the_reset():
|
||||
"""Read it after the reset and every interruption reports zeroed counters —
|
||||
which reads like hard evidence and is nothing of the sort."""
|
||||
detector = FakeBargeIn()
|
||||
details = ["score 0.81 at frame 12", "score 0.000 at frame 0"]
|
||||
speaker, state, *_ = build(barge_in=lambda: detector,
|
||||
detail_of=lambda: details.pop(0))
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("Hello there."))
|
||||
|
||||
assert speaker.last_detail == "score 0.81 at frame 12"
|
||||
|
||||
|
||||
def test_a_detector_that_appears_late_is_still_used():
|
||||
"""The detector is built after the speaker — it needs the mic stream — so
|
||||
it's read through a callable. Holding a copy is how the two drift apart."""
|
||||
detector = None
|
||||
speaker, state, tts, *_ = build(barge_in=lambda: detector)
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("Before."))
|
||||
assert tts.calls[-1]["interruptible"] is False
|
||||
|
||||
detector = FakeBargeIn()
|
||||
speaker.say(Utterance.reply("After."))
|
||||
assert tts.calls[-1]["interruptible"] is True
|
||||
|
||||
|
||||
# ── the mouth ───────────────────────────────────────────────────────────────
|
||||
|
||||
def test_the_mouth_is_closed_when_the_line_ends():
|
||||
"""A pet left mid-vowel after the audio stops looks broken."""
|
||||
speaker, state, _tts, _said, _logged, _recorded, levels = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("Talking."))
|
||||
|
||||
assert levels[0] > 0
|
||||
assert levels[-1] == 0.0
|
||||
|
||||
|
||||
# ── text handling ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_an_empty_line_is_not_spoken_at_all():
|
||||
"""A reply that is nothing but an audio tag reduces to '' — and a silent
|
||||
bubble with no audio is better than the pet announcing "laughing"."""
|
||||
speaker, state, tts, said, *_ = build()
|
||||
|
||||
assert speaker.say(Utterance.reply("[laughing]")) is True
|
||||
assert tts.calls == []
|
||||
assert said == []
|
||||
assert state.state == PetState.IDLE
|
||||
|
||||
|
||||
def test_the_bubble_gets_display_text_and_tts_gets_the_original():
|
||||
"""The bubble keeps emoji and drops markdown; TTS does its own stripping
|
||||
(inside speak(), so every path is covered) and needs the real text."""
|
||||
speaker, state, tts, said, *_ = build()
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("**Sixty** percent 🎉"))
|
||||
|
||||
assert said == ["Sixty percent 🎉"]
|
||||
assert tts.calls[-1]["text"] == "**Sixty** percent 🎉"
|
||||
|
||||
|
||||
def test_a_voice_override_reaches_tts():
|
||||
speaker, state, tts, *_ = build(voice_id=lambda: "voice-abc")
|
||||
state.transition(PetState.LISTENING)
|
||||
state.transition(PetState.THINKING)
|
||||
|
||||
speaker.say(Utterance.reply("In character."))
|
||||
|
||||
assert tts.calls[-1]["voice_id"] == "voice-abc"
|
||||
|
||||
|
||||
# ── the follow-up rule ──────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def follow_ups_on(monkeypatch):
|
||||
monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", True)
|
||||
monkeypatch.setattr(config, "FOLLOW_UP_MAX_TURNS", 3)
|
||||
|
||||
|
||||
def test_an_interruption_always_reopens_the_mic(follow_ups_on):
|
||||
"""You talked over it — you are mid-sentence, so it has to listen."""
|
||||
keep, why = speech.follow_up_decision("Anything else?", completed=False, follow_ups=99)
|
||||
assert keep is True
|
||||
assert why == "interrupted"
|
||||
|
||||
|
||||
def test_a_question_keeps_the_mic_open_without_the_wake_word(follow_ups_on):
|
||||
keep, why = speech.follow_up_decision("Want me to check?", completed=True, follow_ups=0)
|
||||
assert (keep, why) == (True, "question")
|
||||
|
||||
|
||||
def test_a_statement_ends_the_turn(follow_ups_on):
|
||||
keep, _why = speech.follow_up_decision("Sixty percent used.", completed=True, follow_ups=0)
|
||||
assert keep is False
|
||||
|
||||
|
||||
def test_the_cap_stops_a_server_that_ends_every_reply_with_a_question(follow_ups_on):
|
||||
"""Otherwise mic noise loops it forever."""
|
||||
assert speech.follow_up_decision("Ok?", completed=True, follow_ups=2)[0] is True
|
||||
keep, why = speech.follow_up_decision("Ok?", completed=True, follow_ups=3)
|
||||
assert keep is False
|
||||
assert "cap" in why
|
||||
|
||||
|
||||
def test_the_rule_can_be_switched_off(monkeypatch):
|
||||
monkeypatch.setattr(config, "FOLLOW_UP_LISTEN", False)
|
||||
assert speech.follow_up_decision("Ok?", completed=True, follow_ups=0)[0] is False
|
||||
@@ -4,6 +4,8 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet.speech_text import for_display, for_speech, is_question
|
||||
@@ -77,3 +79,33 @@ def test_is_question_ignores_question_marks_that_are_not_spoken():
|
||||
assert not is_question("Docs are at https://example.com/x?y=1")
|
||||
assert not is_question("")
|
||||
assert not is_question(None)
|
||||
|
||||
|
||||
# ── ElevenLabs v3 delivery tags (observed live 2026-07-31) ──────────────────
|
||||
# "[laughing] That one came through clean" was spoken as "laughing That one
|
||||
# came through clean": the tags mean something to the dialogue model inside a
|
||||
# dialoguectl scene, and nothing at all to the ordinary reply voice.
|
||||
|
||||
def test_delivery_tags_are_never_spoken_aloud():
|
||||
spoken = for_speech("[laughing] That one came through clean.")
|
||||
assert "laughing" not in spoken
|
||||
assert spoken.startswith("That one came through clean")
|
||||
|
||||
|
||||
def test_the_bubble_does_not_caption_a_laugh_nobody_heard():
|
||||
assert "[laughing]" not in for_display("[laughing] All good.")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tag", [
|
||||
"[whispering]", "[cheerfully]", "[sighs]", "[laughs]", "[clears throat]",
|
||||
"[nervously]", "[excited]", "[pause]", "[shouting]",
|
||||
])
|
||||
def test_the_common_tags_are_all_covered(tag):
|
||||
assert "" == for_speech(tag).strip()
|
||||
|
||||
|
||||
def test_ordinary_bracketed_text_survives():
|
||||
"""The tags are matched narrowly on purpose — real bracketed content is
|
||||
part of what the user asked to hear."""
|
||||
assert "1" in for_speech("See reference [1] for details.")
|
||||
assert "docs" in for_speech("It's in [the docs] somewhere.")
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Streaming speech-to-text: the protocol, and the fallback that makes it safe
|
||||
to switch on at all.
|
||||
|
||||
A fake websocket throughout — no network, no Deepgram account, no audio.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet.audio import mic
|
||||
from bolt_pet.audio.stt_stream import StreamingTranscriber
|
||||
|
||||
|
||||
class FakeSocket:
|
||||
"""Records what was sent; replays scripted Deepgram frames."""
|
||||
|
||||
def __init__(self, messages=(), fail_on_send=False):
|
||||
self.sent = []
|
||||
self.closed = False
|
||||
self.fail_on_send = fail_on_send
|
||||
self._messages = list(messages)
|
||||
|
||||
def send_binary(self, data):
|
||||
if self.fail_on_send:
|
||||
raise ConnectionError("socket died")
|
||||
self.sent.append(data)
|
||||
|
||||
def send(self, text):
|
||||
self.sent.append(text)
|
||||
|
||||
def recv(self):
|
||||
if self._messages:
|
||||
return self._messages.pop(0)
|
||||
time.sleep(0.01)
|
||||
raise ConnectionError("closed")
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _results(transcript, is_final=True):
|
||||
return json.dumps({
|
||||
"type": "Results", "is_final": is_final,
|
||||
"channel": {"alternatives": [{"transcript": transcript}]},
|
||||
})
|
||||
|
||||
|
||||
def _frame(value=1000):
|
||||
return np.full(320, value, dtype=np.int16)
|
||||
|
||||
|
||||
# ── the protocol ────────────────────────────────────────────────────────────
|
||||
|
||||
def test_frames_go_up_as_they_are_captured():
|
||||
socket = FakeSocket([_results("what's the weather")])
|
||||
session = StreamingTranscriber(socket)
|
||||
|
||||
for _ in range(3):
|
||||
session.feed(_frame())
|
||||
text = session.finish()
|
||||
|
||||
assert len(socket.sent) == 4 # three frames plus the close message
|
||||
assert text == "what's the weather"
|
||||
assert socket.closed
|
||||
|
||||
|
||||
def test_only_final_results_are_kept():
|
||||
"""Interim hypotheses change under you; concatenating them would produce
|
||||
"what what's what's the what's the weather"."""
|
||||
socket = FakeSocket([
|
||||
_results("what's", is_final=False),
|
||||
_results("what's the", is_final=False),
|
||||
_results("what's the weather", is_final=True),
|
||||
])
|
||||
session = StreamingTranscriber(socket)
|
||||
assert session.finish() == "what's the weather"
|
||||
|
||||
|
||||
def test_several_final_segments_are_joined():
|
||||
socket = FakeSocket([_results("turn the lights on"), _results("in the kitchen")])
|
||||
session = StreamingTranscriber(socket)
|
||||
assert session.finish() == "turn the lights on in the kitchen"
|
||||
|
||||
|
||||
def test_junk_frames_are_ignored_rather_than_killing_the_reader():
|
||||
"""An exception on the reader thread would silently end transcription for
|
||||
the rest of the utterance."""
|
||||
socket = FakeSocket(["not json at all", '{"type":"Metadata"}',
|
||||
_results("still works")])
|
||||
session = StreamingTranscriber(socket)
|
||||
assert session.finish() == "still works"
|
||||
|
||||
|
||||
def test_an_empty_frame_means_the_socket_closed():
|
||||
"""websocket-client returns "" from recv() on a closed connection, so it
|
||||
ends the read loop rather than being treated as a blank transcript."""
|
||||
socket = FakeSocket([_results("heard this much"), "", _results("never arrives")])
|
||||
session = StreamingTranscriber(socket)
|
||||
assert session.finish() == "heard this much"
|
||||
|
||||
|
||||
def test_a_socket_that_dies_mid_utterance_gives_up_quietly():
|
||||
socket = FakeSocket([], fail_on_send=True)
|
||||
session = StreamingTranscriber(socket)
|
||||
|
||||
session.feed(_frame()) # must not raise — recording carries on
|
||||
assert session.finish() == ""
|
||||
|
||||
|
||||
# ── opening: failure is an ordinary outcome ────────────────────────────────
|
||||
|
||||
def test_open_returns_none_when_it_cannot_connect():
|
||||
"""None means "the one-shot path will do it", not an error."""
|
||||
def refuse():
|
||||
raise OSError("no network")
|
||||
|
||||
assert StreamingTranscriber.open(connect=refuse) is None
|
||||
|
||||
|
||||
def test_open_returns_a_session_when_it_can():
|
||||
session = StreamingTranscriber.open(connect=lambda: FakeSocket([_results("hi")]))
|
||||
assert session is not None
|
||||
assert session.finish() == "hi"
|
||||
|
||||
|
||||
def test_streaming_is_off_without_the_switch_or_a_key(monkeypatch):
|
||||
from bolt_pet.audio import stt_stream
|
||||
|
||||
monkeypatch.setattr(stt_stream.config, "STT_STREAMING", False)
|
||||
assert stt_stream.available() is False
|
||||
monkeypatch.setattr(stt_stream.config, "STT_STREAMING", True)
|
||||
monkeypatch.setattr(stt_stream.config, "DEEPGRAM_API_KEY", "")
|
||||
assert stt_stream.available() is False
|
||||
|
||||
|
||||
# ── the capture hook ────────────────────────────────────────────────────────
|
||||
|
||||
class _Stream:
|
||||
"""Loud frames, then quiet ones, so the VAD ends the utterance."""
|
||||
|
||||
def __init__(self, loud=6, quiet=40):
|
||||
self.frames = ([np.full((320, 1), 3000, dtype=np.int16)] * loud
|
||||
+ [np.zeros((320, 1), dtype=np.int16)] * quiet)
|
||||
|
||||
def read(self, n):
|
||||
return (self.frames.pop(0) if self.frames
|
||||
else np.zeros((320, 1), dtype=np.int16)), None
|
||||
|
||||
|
||||
def test_recording_hands_every_speech_frame_to_the_listener():
|
||||
seen = []
|
||||
pcm = mic.record_utterance(_Stream(), on_frame=seen.append,
|
||||
silence_end_sec=0.2, min_utterance_s=0.0)
|
||||
assert pcm is not None
|
||||
assert len(seen) >= 6 # every frame of speech was streamed
|
||||
|
||||
|
||||
def test_a_listener_that_throws_cannot_break_the_recording():
|
||||
"""The fallback is about to need this audio — a dead stream must not cost
|
||||
the recording too."""
|
||||
def explode(frame):
|
||||
raise RuntimeError("stream died")
|
||||
|
||||
pcm = mic.record_utterance(_Stream(), on_frame=explode,
|
||||
silence_end_sec=0.2, min_utterance_s=0.0)
|
||||
assert pcm is not None and len(pcm) > 0
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Remembering where the pet was left — and refusing to when that's a trap."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from bolt_pet import window_state
|
||||
|
||||
|
||||
def test_a_saved_position_comes_back(tmp_path):
|
||||
path = tmp_path / "window.json"
|
||||
window_state.save(1200, 640, path)
|
||||
assert window_state.load(path) == (1200, 640)
|
||||
|
||||
|
||||
def test_no_file_yet_is_not_an_error(tmp_path):
|
||||
assert window_state.load(tmp_path / "nope.json") is None
|
||||
|
||||
|
||||
def test_a_corrupt_file_falls_back_to_the_default_corner(tmp_path):
|
||||
"""Whatever is in there, the pet still has to start."""
|
||||
path = tmp_path / "window.json"
|
||||
for junk in ("", "{", "null", "[]", '{"x": "left"}', '{"y": 3}'):
|
||||
path.write_text(junk)
|
||||
assert window_state.load(path) is None
|
||||
|
||||
|
||||
def test_saving_creates_the_cache_directory(tmp_path):
|
||||
path = tmp_path / "deep" / "nested" / "window.json"
|
||||
window_state.save(10, 20, path)
|
||||
assert window_state.load(path) == (10, 20)
|
||||
|
||||
|
||||
def test_an_unwritable_location_is_swallowed(tmp_path):
|
||||
"""A read-only cache dir is a reason to forget the position, not to crash
|
||||
on every drag."""
|
||||
window_state.save(1, 2, tmp_path / "no" / "\0bad" / "window.json")
|
||||
|
||||
|
||||
def test_the_write_is_atomic_and_leaves_no_litter(tmp_path):
|
||||
path = tmp_path / "window.json"
|
||||
window_state.save(5, 5, path)
|
||||
window_state.save(7, 7, path)
|
||||
assert json.loads(path.read_text()) == {"x": 7, "y": 7}
|
||||
assert [p.name for p in tmp_path.iterdir()] == ["window.json"]
|
||||
|
||||
|
||||
# ── validating against the screens that exist *now* ─────────────────────────
|
||||
|
||||
LAPTOP = (0, 0, 1920, 1080)
|
||||
EXTERNAL = (1920, 0, 4480, 1440)
|
||||
|
||||
|
||||
def test_a_position_on_a_connected_screen_is_kept():
|
||||
assert window_state.is_visible_on(1700, 900, 128, [LAPTOP])
|
||||
|
||||
|
||||
def test_a_position_on_an_unplugged_monitor_is_refused():
|
||||
"""The dangerous case: restoring it faithfully puts the pet somewhere you
|
||||
can't see or reach."""
|
||||
assert not window_state.is_visible_on(3000, 700, 128, [LAPTOP])
|
||||
assert window_state.is_visible_on(3000, 700, 128, [LAPTOP, EXTERNAL])
|
||||
|
||||
|
||||
def test_mostly_off_screen_counts_as_gone():
|
||||
"""A few pixels of ear poking onto the desktop is not "reachable"."""
|
||||
assert not window_state.is_visible_on(1910, 500, 128, [LAPTOP])
|
||||
assert window_state.is_visible_on(1830, 500, 128, [LAPTOP])
|
||||
|
||||
|
||||
def test_touching_an_edge_is_not_overlapping():
|
||||
assert not window_state.is_visible_on(1920, 0, 128, [LAPTOP])
|
||||
|
||||
|
||||
def test_negative_coordinates_are_fine_when_a_screen_is_there():
|
||||
"""Monitors left of or above the primary have negative origins."""
|
||||
left_of_primary = (-1920, 0, 0, 1080)
|
||||
assert window_state.is_visible_on(-900, 400, 128, [left_of_primary, LAPTOP])
|
||||
|
||||
|
||||
def test_no_screens_at_all_is_not_visible():
|
||||
assert not window_state.is_visible_on(100, 100, 128, [])
|
||||
Reference in New Issue
Block a user