Keep listening while an utterance is being transcribed

stt.finish() used to run inline in the frame loop, deafening the
assistant for up to ~4s per utterance while the server returned the
transcript (and whatever was said meanwhile was later drained). The
finish now runs on a background thread per utterance; the segmenter
keeps consuming mic frames, so speech during the wait becomes its own
utterance and is transcribed too. Transcripts are delivered oldest-
first between frames, deferred while an utterance is in progress, and
the command/follow-up window no longer times out on transcription
latency. Speaking stays half-duplex (echo protection).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-10-04 22:14:42 -06:00
co-authored by Claude Fable 5
parent 34b2d302f9
commit ca65670638
2 changed files with 95 additions and 16 deletions
+29 -4
View File
@@ -111,19 +111,22 @@ def test_long_speech_is_cut_off_at_the_cap():
# ── the flow ────────────────────────────────────────────────────────────────
class FakeTranscription:
"""Each utterance returns the next scripted transcript."""
"""Each utterance takes the next scripted transcript. Bound at creation
(the "start" event, on the main thread) because finish() now runs on a
worker thread per utterance and two may be in flight at once."""
script = []
def __init__(self):
self.frames = 0
self.cancelled = False
self.transcript = FakeTranscription.script.pop(0)
def add(self, frames):
self.frames += len(frames)
def finish(self, audio):
return FakeTranscription.script.pop(0)
return self.transcript
def cancel(self):
self.cancelled = True
@@ -141,7 +144,7 @@ def utterance():
return [frame(LOUD)] * 8 + [frame(QUIET)] * 12
def make(replies, commands="read", streaming=False, **kw):
def make(replies, commands="read", streaming=False, transcription=FakeTranscription, **kw):
asked, played, logs = [], [], []
def converse(message, on_command, on_say=None):
@@ -152,7 +155,7 @@ def make(replies, commands="read", streaming=False, **kw):
return server.Reply(reply)
clock = Clock()
a = Assistant(transcription=FakeTranscription, converse=converse,
a = Assistant(transcription=transcription, converse=converse,
converse_stream=None if not streaming else kw.pop("stream"),
tts=lambda text, voice: [text.encode()], play=lambda chunks: played.extend(chunks),
tools=Tools(mode=commands, log=logs.append), log=logs.append, clock=clock, **kw)
@@ -163,6 +166,7 @@ def say(a, transcript):
FakeTranscription.script.append(transcript)
for f in utterance():
a.process(f)
a.handle_transcripts(wait=True) # what run() does between frames
def spoken(played):
@@ -225,6 +229,27 @@ def test_mic_audio_during_speech_is_discarded():
assert not a.segmenter.in_speech
def test_listening_continues_while_a_transcript_is_in_flight():
import threading
release = threading.Event()
class Slow(FakeTranscription):
def finish(self, audio):
release.wait(timeout=5)
return self.transcript
a, asked, played, logs, _ = make(["One.", "Two."], transcription=Slow)
FakeTranscription.script += ["hey bolt first thing", "hey bolt second thing"]
for f in utterance() + utterance(): # the second said while the first transcribes
a.process(f)
assert asked == [] # both transcripts still at the "server"
release.set()
a.handle_transcripts(wait=True)
assert len(asked) == 2 # nothing said during the wait was lost
assert asked[0].startswith("first thing") and asked[1].startswith("second thing")
def test_streamed_sentences_are_spoken_as_they_arrive():
def stream(message, on_say, on_command):
on_say("First.")