Keep listening while an utterance is being transcribed
stt.finish() used to run inline in the frame loop, deafening the assistant for up to ~4s per utterance while the server returned the transcript (and whatever was said meanwhile was later drained). The finish now runs on a background thread per utterance; the segmenter keeps consuming mic frames, so speech during the wait becomes its own utterance and is transcribed too. Transcripts are delivered oldest- first between frames, deferred while an utterance is in progress, and the command/follow-up window no longer times out on transcription latency. Speaking stays half-duplex (echo protection). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+29
-4
@@ -111,19 +111,22 @@ def test_long_speech_is_cut_off_at_the_cap():
|
||||
# ── the flow ────────────────────────────────────────────────────────────────
|
||||
|
||||
class FakeTranscription:
|
||||
"""Each utterance returns the next scripted transcript."""
|
||||
"""Each utterance takes the next scripted transcript. Bound at creation
|
||||
(the "start" event, on the main thread) because finish() now runs on a
|
||||
worker thread per utterance and two may be in flight at once."""
|
||||
|
||||
script = []
|
||||
|
||||
def __init__(self):
|
||||
self.frames = 0
|
||||
self.cancelled = False
|
||||
self.transcript = FakeTranscription.script.pop(0)
|
||||
|
||||
def add(self, frames):
|
||||
self.frames += len(frames)
|
||||
|
||||
def finish(self, audio):
|
||||
return FakeTranscription.script.pop(0)
|
||||
return self.transcript
|
||||
|
||||
def cancel(self):
|
||||
self.cancelled = True
|
||||
@@ -141,7 +144,7 @@ def utterance():
|
||||
return [frame(LOUD)] * 8 + [frame(QUIET)] * 12
|
||||
|
||||
|
||||
def make(replies, commands="read", streaming=False, **kw):
|
||||
def make(replies, commands="read", streaming=False, transcription=FakeTranscription, **kw):
|
||||
asked, played, logs = [], [], []
|
||||
|
||||
def converse(message, on_command, on_say=None):
|
||||
@@ -152,7 +155,7 @@ def make(replies, commands="read", streaming=False, **kw):
|
||||
return server.Reply(reply)
|
||||
|
||||
clock = Clock()
|
||||
a = Assistant(transcription=FakeTranscription, converse=converse,
|
||||
a = Assistant(transcription=transcription, converse=converse,
|
||||
converse_stream=None if not streaming else kw.pop("stream"),
|
||||
tts=lambda text, voice: [text.encode()], play=lambda chunks: played.extend(chunks),
|
||||
tools=Tools(mode=commands, log=logs.append), log=logs.append, clock=clock, **kw)
|
||||
@@ -163,6 +166,7 @@ def say(a, transcript):
|
||||
FakeTranscription.script.append(transcript)
|
||||
for f in utterance():
|
||||
a.process(f)
|
||||
a.handle_transcripts(wait=True) # what run() does between frames
|
||||
|
||||
|
||||
def spoken(played):
|
||||
@@ -225,6 +229,27 @@ def test_mic_audio_during_speech_is_discarded():
|
||||
assert not a.segmenter.in_speech
|
||||
|
||||
|
||||
def test_listening_continues_while_a_transcript_is_in_flight():
|
||||
import threading
|
||||
|
||||
release = threading.Event()
|
||||
|
||||
class Slow(FakeTranscription):
|
||||
def finish(self, audio):
|
||||
release.wait(timeout=5)
|
||||
return self.transcript
|
||||
|
||||
a, asked, played, logs, _ = make(["One.", "Two."], transcription=Slow)
|
||||
FakeTranscription.script += ["hey bolt first thing", "hey bolt second thing"]
|
||||
for f in utterance() + utterance(): # the second said while the first transcribes
|
||||
a.process(f)
|
||||
assert asked == [] # both transcripts still at the "server"
|
||||
release.set()
|
||||
a.handle_transcripts(wait=True)
|
||||
assert len(asked) == 2 # nothing said during the wait was lost
|
||||
assert asked[0].startswith("first thing") and asked[1].startswith("second thing")
|
||||
|
||||
|
||||
def test_streamed_sentences_are_spoken_as_they_arrive():
|
||||
def stream(message, on_say, on_command):
|
||||
on_say("First.")
|
||||
|
||||
Reference in New Issue
Block a user