Bolt Assistant: wake-phrase voice assistant for Bolt

bolt_assistant/__init__.py

@@ -0,0 +1,3 @@
+"""Bolt Assistant: a voice assistant for Bolt (wake phrase, server speech-to-text, spoken replies)."""
+
+__version__ = "0.1.0"

bolt_assistant/__main__.py

@@ -0,0 +1,117 @@
+"""Bolt Assistant: say "Hey Bolt", ask, hear the answer.
+
+    python -m bolt_assistant                 listen on the default mic
+    python -m bolt_assistant --verbose       also show what was heard but ignored
+    python -m bolt_assistant --type          type requests instead of speaking (testing)
+    python -m bolt_assistant --say "hello"   speak a line through the speaker and exit
+    python -m bolt_assistant --devices       list microphones and speakers
+    python -m bolt_assistant --check         check settings, server, mic and voice
+"""
+
+from __future__ import annotations
+
+import sys
+import threading
+
+from . import __version__, audio, config, server
+
+
+def _devices() -> int:
+    import sounddevice as sd
+
+    print(sd.query_devices())
+    print("\nDefault input/output:", sd.default.device,
+          "\nSet MIC_DEVICE / SPEAKER_DEVICE in .env to an index or a name to choose another.")
+    return 0
+
+
+def _check() -> int:
+    from .stt import connect
+
+    ok = True
+
+    def row(name, good, detail):
+        nonlocal ok
+        ok = ok and good
+        print(f"[{' ok ' if good else 'FAIL'}] {name:12} {detail}")
+
+    missing = config.missing_config()
+    row("settings", not missing, "all set" if not missing else "missing " + ", ".join(missing))
+    try:
+        server.check_health()
+        row("server", True, config.SERVER_URL)
+    except Exception as exc:
+        row("server", False, f"{config.SERVER_URL}: {exc}")
+    try:
+        import sounddevice as sd
+
+        mic = sd.query_devices(config.MIC_DEVICE, "input")["name"]
+        out = sd.query_devices(config.SPEAKER_DEVICE, "output")["name"]
+        row("microphone", True, mic)
+        row("speaker", True, out)
+    except Exception as exc:
+        row("audio", False, f"{exc} (pip install sounddevice; Linux: libportaudio2)")
+    try:
+        connect().close()
+        row("speech-to-text", True, "/desk/stt reachable")
+    except Exception as exc:
+        row("speech-to-text", False, f"{exc} (pip install websocket-client)")
+    row("commands", True, f"ASSISTANT_COMMANDS={config.COMMANDS}")
+    return 0 if ok else 1
+
+
+def main(argv=None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    for stream in (sys.stdout, sys.stderr):
+        try:  # replies carry dashes and curly quotes; a cp1252 console must not crash on them
+            stream.reconfigure(errors="replace")
+        except Exception:
+            pass
+    if "-h" in argv or "--help" in argv:
+        print(__doc__.strip())
+        return 0
+    if "--devices" in argv:
+        return _devices()
+    if "--check" in argv:
+        return _check()
+    missing = config.missing_config()
+    if missing:
+        print(f"Missing {', '.join(missing)}. Copy .env.example to .env and fill them in.")
+        return 2
+
+    from .assistant import Assistant
+
+    assistant = Assistant(verbose="--verbose" in argv)
+    print(f"Bolt Assistant {__version__}  |  {config.SERVER_URL}  |  session {config.SESSION_ID}")
+
+    if "--say" in argv:
+        assistant.speak(" ".join(argv[argv.index("--say") + 1:]) or "Hello, I'm Bolt.")
+        return 0
+
+    if "--type" in argv:  # the whole pipeline minus the microphone
+        print("Type what you'd say (including the wake phrase). Ctrl+C to quit.")
+        try:
+            while True:
+                assistant.heard(input("say> "))
+        except (EOFError, KeyboardInterrupt):
+            return 0
+
+    stop = threading.Event()
+    try:
+        mic = audio.open_mic(assistant.on_mic_frame)
+    except Exception as exc:
+        print(f"Couldn't open the microphone: {exc}\nTry --devices, and MIC_DEVICE in .env.")
+        return 1
+    try:
+        assistant.run(stop)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        stop.set()
+        mic.stop()
+        mic.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bolt_assistant/assistant.py

@@ -0,0 +1,259 @@
+"""The assistant: hear the wake phrase, take the request, answer out loud.
+
+    idle      every utterance the local detector passes on is transcribed by
+              the server; only ones that open with a wake phrase count
+                "Hey Bolt, what's the time"  -> ask "what's the time"
+                "Hey Bolt"                   -> chime, go to `command`
+    command   the next thing said (within COMMAND_WAIT_SEC) is the request
+    follow_up the reply ended on a question: the next thing said (within
+              FOLLOW_UP_SEC) is the answer, no wake phrase needed
+
+Speaking is half-duplex: the mic is ignored while Bolt talks and for a
+moment after, so his own voice through the speakers can't wake him (his
+replies say "Bolt" often enough).
+
+Everything that touches hardware or the network is injected, so the flow
+is tested with fakes; __main__ wires in the real mic, server and speaker.
+"""
+
+from __future__ import annotations
+
+import platform
+import queue
+import threading
+import time
+from typing import Callable, Optional
+
+from . import audio, config, server, speech_text
+from .listener import Segmenter
+from .stt import SttError, Transcription
+from .tools import Tools
+from .wake import find_wake
+
+ECHO_GUARD_SEC = 0.4  # ignore the mic this long after speaking
+
+
+def os_label() -> str:
+    try:
+        return f"{platform.system()} {platform.release()}".strip() or "a computer"
+    except Exception:
+        return "a computer"
+
+
+def annotate(text: str, commands: str) -> str:
+    """The note on every request: it's voice, and what Bolt may do here."""
+    can = {
+        "off": "You can't act on this computer from here.",
+        "read": ('You can list and read files with your command tool, e.g. '
+                 '`command: filectl {"op": "list", "path": "..."}` (ops: list, read), and the real '
+                 "output comes back. You can't run shell commands or change files from here."),
+        "all": ('You can act on this computer with your command tool: `command: filectl {"op": ...}` '
+                "for files (list, read, write, edit) or `command: <shell command>`; the real "
+                "output comes back."),
+    }[commands]
+    return (f"{text}\n\n[Bolt Assistant: a voice-only assistant on {os_label()}. The request was "
+            "spoken and transcribed, so it may contain recognition mistakes. Your reply is read "
+            "aloud: keep it short and conversational, with no markdown, lists, links or code. "
+            f"{can} Never make up a command's output.]")
+
+
+class Assistant:
+    def __init__(
+        self,
+        *,
+        transcription: Callable = Transcription,
+        converse: Callable = server.converse,
+        converse_stream: Optional[Callable] = server.converse_stream,
+        tts: Callable = server.tts_stream,
+        play: Callable = audio.play,
+        tools: Optional[Tools] = None,
+        log: Callable[[str], None] = print,
+        clock: Callable[[], float] = time.monotonic,
+        wake_words=None,
+        verbose: bool = False,
+    ):
+        self._transcription = transcription
+        self._converse = converse
+        self._stream = converse_stream if config.STREAMING_REPLIES else None
+        self._tts, self._play = tts, play
+        self.log, self.clock, self.verbose = log, clock, verbose
+        self.tools = tools or Tools(log=log)
+        self.wake_words = wake_words or config.WAKE_WORDS
+        self.segmenter = Segmenter(config.VAD_RMS_THRESHOLD, config.VAD_START_FRAMES,
+                                   config.VAD_SILENCE_END_SEC, config.MIN_UTTERANCE_SEC,
+                                   config.MAX_UTTERANCE_SEC)
+        self.frames: "queue.Queue[bytes]" = queue.Queue(maxsize=2000)  # ~160 s of audio
+        self.state = "idle"
+        self._deadline = 0.0
+        self._follow_ups = 0
+        self._stt = None
+        self._mute_until = 0.0
+        self._voice = ""  # a voice the server picked (speak_as) sticks for later replies
+        self._tts_failed = False
+
+    # ── audio in ─────────────────────────────────────────────────────────
+
+    def on_mic_frame(self, frame: bytes) -> None:
+        """Called from the audio thread: queue it, never block."""
+        try:
+            self.frames.put_nowait(frame)
+        except queue.Full:
+            pass  # we've fallen far behind; dropping audio beats unbounded memory
+
+    def run(self, stop: threading.Event) -> None:
+        self.log(self.listening_banner())
+        while not stop.is_set():
+            try:
+                frame = self.frames.get(timeout=0.2)
+            except queue.Empty:
+                self._check_deadline()
+                continue
+            self.process(frame)
+
+    def listening_banner(self) -> str:
+        return 'Listening for "Hey Bolt" (or "Thunderbolt", "Hey Assistant"). Ctrl+C to quit.'
+
+    def process(self, frame: bytes) -> None:
+        if self.clock() < self._mute_until:
+            return
+        self._check_deadline()
+        for event in self.segmenter.feed(frame):
+            if event.kind == "start":
+                self._stt = self._transcription()
+                self._stt.add(event.frames)
+            elif event.kind == "frame" and self._stt is not None:
+                self._stt.add(event.frames)
+            elif event.kind == "dropped":
+                if self._stt is not None:
+                    self._stt.cancel()
+                self._stt = None
+            elif event.kind == "end":
+                stt, self._stt = self._stt, None
+                try:
+                    text = stt.finish(event.audio) if stt is not None else ""
+                except SttError as exc:
+                    self.log(f"! transcription failed: {exc}")
+                    if self.state != "idle":
+                        self._chime(audio.ERROR_CHIME)
+                    continue
+                self.heard(text)
+
+    def _check_deadline(self) -> None:
+        if self.state in ("command", "follow_up") and self.clock() > self._deadline \
+                and not self.segmenter.in_speech:
+            if self.verbose:
+                self.log("  (stopped waiting)")
+            self.state = "idle"
+
+    # ── what was said ────────────────────────────────────────────────────
+
+    def heard(self, text: str) -> None:
+        text = (text or "").strip()
+        if not text:
+            return
+        self._check_deadline()
+        woke, rest = find_wake(text, self.wake_words)
+        if self.state == "idle":
+            if not woke:
+                if self.verbose:
+                    self.log(f"  (not for me) {text}")
+                return
+            if not rest:
+                self.log("* Yes?")
+                self._chime(audio.WAKE_CHIME)
+                self._wait("command", config.COMMAND_WAIT_SEC)
+                return
+            self._follow_ups = 0
+            self.ask(rest)
+            return
+        # command / follow_up: whatever was said is the request. A repeated
+        # "Hey Bolt" in front of it is dropped; on its own it just re-arms.
+        if woke and not rest:
+            self._chime(audio.WAKE_CHIME)
+            self._wait(self.state, config.COMMAND_WAIT_SEC)
+            return
+        if self.state == "command":
+            self._follow_ups = 0
+        self.ask(rest if woke else text)
+
+    def _wait(self, state: str, seconds: float) -> None:
+        self.state = state
+        self._deadline = self.clock() + seconds
+
+    # ── asking and answering ─────────────────────────────────────────────
+
+    def ask(self, request: str) -> None:
+        self.state = "busy"
+        self.log(f"> {request}")
+        spoken = []
+
+        def say(sentence: str) -> None:
+            spoken.append(sentence)
+            self.speak(sentence)
+
+        message = annotate(request, self.tools.mode)
+        try:
+            reply = None
+            if self._stream is not None:
+                try:
+                    reply = self._stream(message, on_say=say, on_command=self.tools)
+                except server.ServerError:
+                    reply = None  # nothing was said yet: the plain path is invisible
+            if reply is None:
+                reply = self._converse(message, on_command=self.tools, on_say=say)
+        except server.ServerError as exc:
+            self.log(f"! {exc}")
+            self._chime(audio.ERROR_CHIME)
+            self._back_to_listening("idle")
+            return
+        if reply.voice_id:
+            self._voice = reply.voice_id
+        if not reply.streamed and reply.text.strip():
+            spoken.append(reply.text)
+            self.speak(reply.text)
+        full = " ".join(spoken)
+        if speech_text.is_question(full) and self._follow_ups < config.FOLLOW_UP_MAX_TURNS:
+            self._follow_ups += 1
+            self._back_to_listening("follow_up", config.FOLLOW_UP_SEC)
+        else:
+            self._follow_ups = 0
+            self._back_to_listening("idle")
+
+    def speak(self, text: str) -> None:
+        words = speech_text.for_speech(text)
+        if not words:
+            return
+        self.log(f"< {speech_text.for_display(text)}")
+        try:
+            self._play(self._tts(words, self._voice))
+            self._tts_failed = False
+        except server.TtsError as exc:
+            if not self._tts_failed:  # say it once, not once per sentence
+                self.log(f"! can't speak: {exc}")
+            self._tts_failed = True
+        except Exception as exc:
+            self.log(f"! playback failed: {exc}")
+
+    def _chime(self, sound: bytes) -> None:
+        if not config.CHIMES:
+            return
+        try:
+            self._play([sound])
+        except Exception:
+            pass
+        self._mute_until = self.clock() + ECHO_GUARD_SEC
+
+    def _back_to_listening(self, state: str, seconds: float = 0.0) -> None:
+        """Drop what the mic picked up while Bolt was talking (his own voice),
+        start detection fresh, and give the room a moment to go quiet."""
+        while True:
+            try:
+                self.frames.get_nowait()
+            except queue.Empty:
+                break
+        self.segmenter.reset()
+        if self._stt is not None:
+            self._stt.cancel()
+            self._stt = None
+        self._mute_until = self.clock() + ECHO_GUARD_SEC
+        self._wait(state, seconds + ECHO_GUARD_SEC)

bolt_assistant/audio.py

@@ -0,0 +1,61 @@
+"""The default microphone in, the default speaker out, and two short chimes.
+
+sounddevice (PortAudio) for both directions; MIC_DEVICE / SPEAKER_DEVICE in
+.env pick something other than the system default. The chimes are made
+here from sine waves — no sound files to ship.
+"""
+
+from __future__ import annotations
+
+import array
+import math
+from typing import Callable, Iterable
+
+from . import config
+
+
+def open_mic(on_frame: Callable[[bytes], None]):
+    """Start the microphone: 16 kHz mono int16, one 80 ms frame per call to
+    *on_frame*, from PortAudio's thread. Returns the stream (stop()/close())."""
+    import sounddevice as sd
+
+    stream = sd.RawInputStream(
+        samplerate=config.SAMPLE_RATE, channels=1, dtype="int16",
+        blocksize=config.FRAME_SAMPLES, device=config.MIC_DEVICE,
+        callback=lambda data, _frames, _time, _status: on_frame(bytes(data)),
+    )
+    stream.start()
+    return stream
+
+
+def play(chunks: Iterable[bytes], sample_rate: int = config.TTS_SAMPLE_RATE,
+         should_stop: Callable[[], bool] = lambda: False) -> bool:
+    """Play PCM16 mono chunks as they arrive. False if stopped early."""
+    import sounddevice as sd
+
+    with sd.RawOutputStream(samplerate=sample_rate, channels=1, dtype="int16",
+                            device=config.SPEAKER_DEVICE) as out:
+        for chunk in chunks:
+            if should_stop():
+                out.abort()
+                return False
+            out.write(chunk)
+    return True
+
+
+def tone(notes, sample_rate: int = config.TTS_SAMPLE_RATE, volume: float = 0.18) -> bytes:
+    """(frequency Hz, seconds) notes as PCM16, each with a quick fade in and
+    out so it doesn't click."""
+    samples = array.array("h")
+    for freq, seconds in notes:
+        count = int(sample_rate * seconds)
+        fade = max(1, int(sample_rate * 0.012))
+        for i in range(count):
+            envelope = min(1.0, i / fade, (count - i) / fade)
+            samples.append(int(32767 * volume * envelope * math.sin(2 * math.pi * freq * i / sample_rate)))
+    return samples.tobytes()
+
+
+# Rising two notes: "I'm listening". Falling: "didn't work".
+WAKE_CHIME = tone([(880, 0.07), (1320, 0.09)])
+ERROR_CHIME = tone([(520, 0.10), (390, 0.14)])

bolt_assistant/config.py

@@ -0,0 +1,110 @@
+"""Settings for Bolt Assistant, from a `.env` next to the project (or the
+real environment, which wins). Everything else reads config from here."""
+
+from __future__ import annotations
+
+import os
+import platform
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent  # project root
+
+
+def _load_env() -> None:
+    env_path = HERE / ".env"
+    if not env_path.exists():
+        return
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        os.environ.setdefault(key.strip(), value.strip())
+
+
+_load_env()
+
+
+def _flag(name: str, default: str) -> bool:
+    return os.environ.get(name, default).strip().lower() in ("1", "true", "yes", "on")
+
+
+def _device(raw: str):
+    """An input/output device: blank = system default, digits = an index,
+    anything else = a name substring (sounddevice's rules)."""
+    raw = (raw or "").strip()
+    if not raw:
+        return None
+    return int(raw) if raw.lstrip("-").isdigit() else raw
+
+
+# ── server ──────────────────────────────────────────────────────────────────
+
+SERVER_URL = os.environ.get("BOLT_SERVER_URL", "").rstrip("/")
+API_KEY = os.environ.get("DESK_API_KEY", "")
+# Its own conversation, separate from the pet's and the console's. The "pet-"
+# prefix gets the desk API's voice-client behaviour (short, speakable replies).
+SESSION_ID = os.environ.get("DESK_SESSION_ID") or "pet-assistant-" + (platform.node() or "desktop")
+# The voice the server's /desk/tts speaks in (an ElevenLabs voice id).
+VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID", "")
+TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "16000"))
+STREAMING_REPLIES = _flag("STREAMING_REPLIES", "true")
+
+# ── audio devices ───────────────────────────────────────────────────────────
+
+MIC_DEVICE = _device(os.environ.get("MIC_DEVICE", ""))
+SPEAKER_DEVICE = _device(os.environ.get("SPEAKER_DEVICE", ""))
+
+# ── wake phrases ────────────────────────────────────────────────────────────
+# Heard through the server's speech-to-text, so these are matched against
+# *text*. A phrase counts only near the start of what was said ("hey bolt,
+# what's the time"), not buried in a sentence ("tighten that bolt"). Speech
+# recognisers often write "Bolt" as "bolts"/"volt"/"bold"; those spellings
+# only count after "hey"/"ok", where they can't be ordinary words.
+
+WAKE_WORDS = [w.strip().lower() for w in os.environ.get(
+    "WAKE_WORDS",
+    "hey bolt,bolt,thunderbolt,hey thunderbolt,ok bolt,okay bolt,yo bolt,hi bolt,"
+    "hey assistant,ok assistant,okay assistant,"
+    "hey bolts,hey volt,hey bold,hey boat,okay volt,hey thunder bolt,thunder bolt",
+).split(",") if w.strip()]
+
+# ── listening (local voice-activity detection, before anything is sent) ────
+# Nothing leaves the machine until this says someone is talking: silence and
+# background hum are never sent (or billed). Raise VAD_RMS_THRESHOLD if
+# background noise keeps getting transcribed; lower it if you have to shout.
+
+VAD_RMS_THRESHOLD = int(os.environ.get("VAD_RMS_THRESHOLD", "500"))
+VAD_START_FRAMES = int(os.environ.get("VAD_START_FRAMES", "3"))           # 80 ms each
+VAD_SILENCE_END_SEC = float(os.environ.get("VAD_SILENCE_END_SEC", "0.8"))
+MIN_UTTERANCE_SEC = float(os.environ.get("MIN_UTTERANCE_SEC", "0.35"))     # shorter: never sent
+MAX_UTTERANCE_SEC = float(os.environ.get("MAX_UTTERANCE_SEC", "12"))
+# After a bare "Hey Bolt", how long to wait for the actual request.
+COMMAND_WAIT_SEC = float(os.environ.get("COMMAND_WAIT_SEC", "8"))
+# When a reply ends on a question, keep listening for the answer without the
+# wake word, for this long, at most this many times in a row.
+FOLLOW_UP_SEC = float(os.environ.get("FOLLOW_UP_SEC", "8"))
+FOLLOW_UP_MAX_TURNS = int(os.environ.get("FOLLOW_UP_MAX_TURNS", "5"))
+CHIMES = _flag("CHIMES", "true")
+
+# ── what Bolt may do on this machine ────────────────────────────────────────
+# The server can relay commands back. With no screen to approve them on:
+#   off   nothing runs; Bolt is told it can't here
+#   read  (default) listing and reading files only, via filectl
+#   all   everything, including shell commands and file writes — like
+#         Bolt-Pet. Only if you trust every request that reaches your mic.
+
+COMMANDS = os.environ.get("ASSISTANT_COMMANDS", "read").strip().lower()
+COMMAND_TIMEOUT_SECONDS = int(os.environ.get("COMMAND_TIMEOUT_SECONDS", "60"))
+
+SAMPLE_RATE = 16000
+FRAME_SAMPLES = 1280  # 80 ms
+
+
+def is_configured() -> bool:
+    return bool(SERVER_URL and API_KEY)
+
+
+def missing_config() -> list:
+    return [name for name, value in (("BOLT_SERVER_URL", SERVER_URL), ("DESK_API_KEY", API_KEY),
+                                     ("ELEVENLABS_VOICE_ID", VOICE_ID)) if not value]

bolt_assistant/file_ops.py

@@ -0,0 +1,287 @@
+"""Local file read/edit/write, intercepted from the server-relayed command
+channel before it reaches a shell (see tools.Tools.handle). Whatever text the server's
+"command" tool sends is just a string this repo is free to interpret before
+it ever reaches subprocess — a `filectl` pseudo-command is one such
+interpretation, giving the model a way to read/write/edit files on this
+machine without constructing a raw shell heredoc, where quoting, `$`,
+backticks, and embedded quotes make anything beyond a one-liner failure-prone.
+
+Executing arbitrary commands already works today (that's exactly what
+run_local_command/subprocess.run does) — filectl doesn't add that ability,
+it only makes the read/write/edit slice of it reliable. It also doesn't
+expand what the server can already do to this machine: a relayed shell
+command could already overwrite any file the desktop user can write (see the
+security notes in the README) — filectl is a safer *path* to the same
+capability, not a new capability.
+
+Wire format: `filectl <json>`, where <json> is a single-line, compact JSON
+object — critically, ONE LINE. The server's "command" tool marker only
+captures the argument up to the next newline (see TOOL_SPECS/
+_extract_all_tool_calls in the main repo's ai/agents/default.py — "command"
+is not declared multiline), so a marker-delimited multi-line payload (this
+module's first design) silently got truncated at the first line no matter
+how the prompt worded it. JSON sidesteps that for free: json.dumps() already
+encodes embedded newlines as the two characters "\n", not an actual line
+break, so arbitrarily multi-line file content still fits on the one physical
+line the extractor captures.
+
+    filectl {"op": "list", "path": "<dir>", "pattern": "<glob>", "recursive": <bool>}
+    filectl {"op": "read", "path": "<path>", "start": <line>, "end": <line>}
+    filectl {"op": "write", "path": "<path>", "content": "<text>"}
+    filectl {"op": "edit", "path": "<path>", "old": "<text>", "new": "<text>"}
+
+"pattern"/"recursive" (list) and "start"/"end" (read) are optional. `list`
+exists even though a real `ls`/`dir` shell command already works, because
+this repo is cross-platform (Windows/macOS/Linux) and the model shouldn't
+have to guess which listing command applies on this machine — one glob-based
+op covers all three. Must be invoked as the argument to the
+ordinary `command` tool marker (e.g. `command: filectl {"op": "write", ...}`)
+— see the pet-only paragraph in the main repo's ai/desk_api.py
+_system_context for the exact instruction the model is given, including the
+"keep it one line" requirement.
+
+Pure parsing (parse) is separated from the filesystem I/O (execute) so the
+syntax is unit-testable without touching disk.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from . import relay_json
+
+_PREFIXES = ("filectl", "file")
+
+# Keeps a runaway read/write/list from blowing up the tool_result relay (and,
+# for read/list, from flooding the model's context with a giant response).
+_MAX_READ_BYTES = 200_000
+_MAX_WRITE_BYTES = 2_000_000
+_MAX_LIST_ENTRIES = 500
+
+HELP = (
+    'filectl {"op": "list", "path": "<dir>", "pattern": "<glob>", "recursive": <bool>}\n'
+    'filectl {"op": "read", "path": "<path>", "start": <line>, "end": <line>}\n'
+    'filectl {"op": "write", "path": "<path>", "content": "<text>"}\n'
+    'filectl {"op": "edit", "path": "<path>", "old": "<text>", "new": "<text>"}\n'
+    "Must be all on one line — this rides the single-line \"command\" marker."
+)
+
+
+class FileOpError(Exception):
+    """Bad filectl syntax or a filesystem error — reported back to the
+    server as command output."""
+
+
+def is_file_command(command: str) -> bool:
+    stripped = (command or "").strip()
+    if not stripped:
+        return False
+    first_word = stripped.split(None, 1)[0]
+    return first_word.lower() in _PREFIXES
+
+
+def parse(command: str) -> Optional[dict]:
+    """Parse a `filectl <json>` string into an action dict, or None if this
+    isn't a filectl command at all (caller should try the next handler, or
+    fall back to a real shell command). Raises FileOpError on a filectl
+    command that doesn't make sense."""
+    if not is_file_command(command):
+        return None
+    stripped = command.strip()
+    _, _, rest = stripped.partition(" ")
+    rest = rest.strip()
+    if not rest or rest.lower() in ("help", "-h", "--help"):
+        return {"action": "help"}
+
+    # Lenient on purpose — see relay_json. A stray quote in machine-written
+    # JSON should not cost a turn, but the repair is reported back so the model
+    # is told it sent something broken while it can still learn from it.
+    try:
+        payload, repairs = relay_json.loads(rest)
+    except relay_json.RelayJsonError as exc:
+        raise FileOpError(f"couldn't parse filectl JSON — {exc}\nusage:\n{HELP}") from exc
+    if not isinstance(payload, dict):
+        raise FileOpError(f"filectl payload must be a JSON object; usage:\n{HELP}")
+
+    op = str(payload.get("op") or "help").lower()
+    # Carried on the action so execute() can tell the model what it got wrong;
+    # a silent repair would fix today's call and guarantee tomorrow's.
+    tag = {"_repairs": repairs} if repairs else {}
+
+    if op == "help":
+        return {"action": "help", **tag}
+
+    if op == "list":
+        path = _required_str(payload, "path")
+        pattern = payload.get("pattern", "*")
+        if not isinstance(pattern, str) or not pattern:
+            raise FileOpError('"pattern" must be a non-empty string')
+        return {
+            "action": "list", "path": path,
+            "pattern": pattern, "recursive": bool(payload.get("recursive")), **tag,
+        }
+
+    if op == "read":
+        path = _required_str(payload, "path")
+        return {
+            "action": "read", "path": path,
+            "start": _line_number(payload.get("start"), "start"),
+            "end": _line_number(payload.get("end"), "end"), **tag,
+        }
+
+    if op == "write":
+        path = _required_str(payload, "path")
+        if payload.get("content") is None:
+            raise FileOpError('write needs "content"')
+        return {"action": "write", "path": path, "content": str(payload["content"]), **tag}
+
+    if op == "edit":
+        path = _required_str(payload, "path")
+        if payload.get("old") is None or payload.get("new") is None:
+            raise FileOpError('edit needs "old" and "new"')
+        old, new = str(payload["old"]), str(payload["new"])
+        if old == new:
+            raise FileOpError("old and new text are identical — nothing to edit")
+        return {"action": "edit", "path": path, "old": old, "new": new, **tag}
+
+    raise FileOpError(f"unknown filectl op {op!r}; usage:\n{HELP}")
+
+
+def _required_str(payload: dict, key: str) -> str:
+    value = payload.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise FileOpError(f'filectl needs a non-empty "{key}"')
+    return value
+
+
+def _line_number(value, label: str) -> Optional[int]:
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise FileOpError(f"{label} must be an integer line number, got {value!r}")
+    return value
+
+
+def describe(action: dict) -> str:
+    """A one-line summary of the action, for logs."""
+    kind = action.get("action")
+    if kind == "help":
+        return "[filectl] help"
+    return f"[filectl] {kind} {action.get('path', '')}"
+
+
+def execute(action: dict) -> str:
+    """Actually perform a parsed filectl action, returning the text to send
+    back to the server as the command's output. Raises FileOpError on any
+    filesystem problem, same as a bad-syntax parse error."""
+    kind = action.get("action")
+    if kind == "help":
+        return HELP
+    if kind == "list":
+        output = _do_list(action)
+    elif kind == "read":
+        output = _do_read(action)
+    elif kind == "write":
+        output = _do_write(action)
+    elif kind == "edit":
+        output = _do_edit(action)
+    else:
+        output = "[filectl] ok"
+    return output + relay_json.repair_note(action.get("_repairs") or [])
+
+
+def _resolve(path_str: str) -> Path:
+    return Path(path_str).expanduser()
+
+
+def _do_list(action: dict) -> str:
+    path = _resolve(action["path"])
+    if not path.is_dir():
+        raise FileOpError(f"no such directory: {path}")
+    pattern = action["pattern"]
+    glob = path.rglob if action["recursive"] else path.glob
+    try:
+        entries = sorted(glob(pattern), key=lambda p: str(p).lower())
+    except OSError as exc:
+        raise FileOpError(f"couldn't list {path}: {exc}") from exc
+    if not entries:
+        return f"[no entries matching {pattern!r} in {path}]"
+    truncated = len(entries) > _MAX_LIST_ENTRIES
+    lines = []
+    for entry in entries[:_MAX_LIST_ENTRIES]:
+        rel = entry.relative_to(path)
+        if entry.is_dir():
+            lines.append(f"{rel}/")
+            continue
+        try:
+            size = entry.stat().st_size
+        except OSError:
+            size = -1
+        lines.append(f"{rel}\t{size}B")
+    if truncated:
+        lines.append(f"... truncated at {_MAX_LIST_ENTRIES} entries (of {len(entries)}) — narrow \"pattern\"")
+    return "\n".join(lines)
+
+
+def _do_read(action: dict) -> str:
+    path = _resolve(action["path"])
+    if not path.is_file():
+        raise FileOpError(f"no such file: {path}")
+    try:
+        data = path.read_bytes()
+    except OSError as exc:
+        raise FileOpError(f"couldn't read {path}: {exc}") from exc
+    if len(data) > _MAX_READ_BYTES:
+        raise FileOpError(
+            f"{path} is {len(data)} bytes, over the {_MAX_READ_BYTES}-byte filectl read limit — "
+            'pass "start"/"end" to read a slice instead'
+        )
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise FileOpError(f"{path} isn't valid UTF-8 text: {exc}") from exc
+    lines = text.splitlines()
+    start = max(1, action.get("start") or 1)
+    end = min(len(lines), action.get("end") or len(lines))
+    if not lines:
+        return "[empty file]"
+    return "\n".join(f"{i:>6}\t{lines[i - 1]}" for i in range(start, end + 1))
+
+
+def _do_write(action: dict) -> str:
+    path = _resolve(action["path"])
+    content = action["content"]
+    if len(content.encode("utf-8")) > _MAX_WRITE_BYTES:
+        raise FileOpError(f"content is over the {_MAX_WRITE_BYTES}-byte filectl write limit")
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content, encoding="utf-8")
+    except OSError as exc:
+        raise FileOpError(f"couldn't write {path}: {exc}") from exc
+    return f"[filectl] wrote {len(content)} chars to {path}"
+
+
+def _do_edit(action: dict) -> str:
+    path = _resolve(action["path"])
+    old, new = action["old"], action["new"]
+    if not path.is_file():
+        raise FileOpError(f"no such file: {path}")
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise FileOpError(f"couldn't read {path}: {exc}") from exc
+    count = text.count(old)
+    if count == 0:
+        raise FileOpError(f"didn't find that text in {path} — nothing changed")
+    if count > 1:
+        raise FileOpError(
+            f"that text appears {count} times in {path} — filectl edit needs a unique match; "
+            "include more surrounding context"
+        )
+    try:
+        path.write_text(text.replace(old, new, 1), encoding="utf-8")
+    except OSError as exc:
+        raise FileOpError(f"couldn't write {path}: {exc}") from exc
+    return f"[filectl] edited {path}"

bolt_assistant/listener.py

@@ -0,0 +1,116 @@
+"""Local voice-activity detection: deciding what's worth sending to the server.
+
+The server's speech-to-text is what recognises "Hey Bolt", which means audio
+has to reach it — but not all of it. `Segmenter` watches loudness frame by
+frame and only opens an utterance when someone seems to be talking:
+
+    VAD_START_FRAMES loud frames in a row  -> speech started
+    VAD_SILENCE_END_SEC of quiet           -> speech ended
+    MAX_UTTERANCE_SEC                       -> cut off regardless
+    shorter than MIN_UTTERANCE_SEC          -> dropped, never sent
+
+Silence, hum and single clicks never leave the machine (or cost anything).
+A short pre-roll is kept so the first syllable — the "H" of "Hey" — isn't
+clipped by the time the detector is sure.
+
+Pure: frames in, events out. No audio devices, no threads, no network.
+"""
+
+from __future__ import annotations
+
+import array
+import math
+from collections import deque
+from typing import List, Optional
+
+PRE_ROLL_FRAMES = 4  # 320 ms kept from before speech was detected
+
+
+def rms(frame: bytes) -> float:
+    samples = array.array("h")
+    samples.frombytes(frame[: len(frame) - len(frame) % 2])
+    return math.sqrt(sum(s * s for s in samples) / len(samples)) if samples else 0.0
+
+
+class Event:
+    """What feed() reports. kind: "start" (frames = pre-roll + the loud
+    frames that triggered it), "frame" (one more frame of an utterance in
+    progress), "end" (audio = the whole utterance), "dropped" (too short
+    to be a wake phrase; nothing was sent)."""
+
+    __slots__ = ("kind", "frames", "audio", "seconds")
+
+    def __init__(self, kind: str, frames: Optional[List[bytes]] = None,
+                 audio: bytes = b"", seconds: float = 0.0):
+        self.kind, self.frames, self.audio, self.seconds = kind, frames or [], audio, seconds
+
+    def __repr__(self) -> str:
+        return f"Event({self.kind!r}, {self.seconds:.2f}s)"
+
+
+class Segmenter:
+    def __init__(self, threshold: float, start_frames: int, silence_end_sec: float,
+                 min_sec: float, max_sec: float, frame_sec: float = 0.08):
+        self.threshold = threshold
+        self.start_frames = max(1, start_frames)
+        self.silence_frames = max(1, round(silence_end_sec / frame_sec))
+        self.min_frames = round(min_sec / frame_sec)
+        self.max_frames = max(1, round(max_sec / frame_sec))
+        self.frame_sec = frame_sec
+        self.reset()
+
+    def reset(self) -> None:
+        """Forget everything — e.g. after the assistant spoke, so its own
+        voice through the speakers isn't taken for the start of a request."""
+        self._pre = deque(maxlen=PRE_ROLL_FRAMES)
+        self._loud_run: List[bytes] = []
+        self._utterance: Optional[List[bytes]] = None
+        self._quiet = 0
+        self._voiced = 0
+
+    @property
+    def in_speech(self) -> bool:
+        return self._utterance is not None
+
+    def feed(self, frame: bytes) -> List[Event]:
+        loud = rms(frame) >= self.threshold
+        if self._utterance is None:
+            return self._idle(frame, loud)
+        return self._speaking(frame, loud)
+
+    def _idle(self, frame: bytes, loud: bool) -> List[Event]:
+        if not loud:
+            if self._loud_run:  # a click or a cough, not speech
+                self._pre.extend(self._loud_run)
+                self._loud_run = []
+            self._pre.append(frame)
+            return []
+        self._loud_run.append(frame)
+        if len(self._loud_run) < self.start_frames:
+            return []
+        frames = list(self._pre) + self._loud_run
+        self._utterance = list(frames)
+        self._voiced, self._quiet = len(self._loud_run), 0
+        self._pre.clear()
+        self._loud_run = []
+        return [Event("start", frames=frames)]
+
+    def _speaking(self, frame: bytes, loud: bool) -> List[Event]:
+        self._utterance.append(frame)
+        if loud:
+            self._voiced += 1
+            self._quiet = 0
+        else:
+            self._quiet += 1
+        events = [Event("frame", frames=[frame])]
+        if self._quiet >= self.silence_frames or len(self._utterance) >= self.max_frames:
+            events.append(self._close())
+        return events
+
+    def _close(self) -> Event:
+        utterance, voiced = self._utterance, self._voiced
+        self._utterance = None
+        seconds = len(utterance) * self.frame_sec
+        if voiced < self.min_frames:
+            return Event("dropped", seconds=seconds)
+        return Event("end", audio=b"".join(utterance), seconds=seconds)

bolt_assistant/relay_json.py

@@ -0,0 +1,157 @@
+"""Lenient JSON for the relayed command channel — and honest about it.
+
+`filectl` and `dialoguectl` both take a single line of compact JSON, hand-typed
+by a language model into a tool marker. Models get that *nearly* right and then
+get it wrong in a small, boringly repeatable set of ways:
+
+    {"op":"list","path":"/home/x","recursive":false"}    ← stray quote after a literal
+    {"op": "read", "path": "/tmp/a.txt",}                ← trailing comma
+    {'op': 'read', 'path': '/tmp/a.txt'}                 ← single quotes
+    {“op”: “read”, “path”: “/tmp/a.txt”}                 ← smart quotes
+    {"op": "list", "recursive": False}                   ← Python literals
+    ```json {"op": "list"} ```                           ← fenced
+
+Observed live (2026-07-30): a stray quote after `false` cost an entire desk
+turn — the call was rejected, the model re-sent the *identical* line, was
+rejected again, and then gave up and told the user "I'll check now" without
+ever calling anything. The user got a promise instead of an answer because of
+one character.
+
+Strict parsing is the wrong trade here. Nothing about a misplaced quote is
+ambiguous, the payload is machine-written and machine-read, and the cost of
+refusing is a wasted round trip that the model has already demonstrated it
+won't recover from. So: try strict first, then apply narrow repairs, and
+accept a repair **only if the result parses**.
+
+Two rules keep this from becoming "guess what they meant":
+
+1. **Repairs are conservative and named.** Each one fixes a known malformation,
+   is applied in isolation, and is reported by name.
+2. **Repairs are never silent.** The caller appends the repair note to the tool
+   result, so the model is told it sent broken JSON *while it still has the
+   turn* — the fix works today and teaches within the conversation. Hiding it
+   would trade a visible failure for an invisible one.
+
+When nothing parses, the error points at the exact character with a caret,
+because "Expecting ',' delimiter: char 74" is not something a model can act on
+and a pointed-at fragment is.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable
+
+# Ordered, cheapest and safest first. Each entry is (name, transform); after
+# each one the payload is re-parsed, so the first repair that works wins and
+# nothing more aggressive gets applied than the input actually needed.
+_REPAIRS: tuple[tuple[str, Callable[[str], str]], ...] = (
+    (
+        "stripped a markdown code fence",
+        lambda text: re.sub(r"^\s*```(?:json)?\s*|\s*```\s*$", "", text),
+    ),
+    (
+        "replaced smart quotes with straight ones",
+        lambda text: text.translate(str.maketrans({"“": '"', "”": '"',
+                                                   "‘": "'", "’": "'"})),
+    ),
+    (
+        "doubled backslashes that weren't valid JSON escapes",
+        # {"path": "C:\Users\x"}  ->  {"path": "C:\\Users\\x"} — a Windows path
+        # typed raw. Only *invalid* escapes are touched, and `\\` pairs are
+        # consumed whole, so nothing that already parsed can change meaning.
+        lambda text: re.sub(
+            r'\\([\\/"bfnrtu])|\\',
+            lambda m: m.group(0) if m.group(1) else "\\\\", text),
+    ),
+    (
+        "removed a stray quote after a bare value",
+        # {"recursive":false"}  ->  {"recursive":false}
+        lambda text: re.sub(
+            r"(:\s*(?:true|false|null|-?\d+(?:\.\d+)?))\s*\"(\s*[,}\]])", r"\1\2", text),
+    ),
+    (
+        "removed a trailing comma",
+        lambda text: re.sub(r",(\s*[}\]])", r"\1", text),
+    ),
+    (
+        "converted Python literals (True/False/None) to JSON",
+        lambda text: re.sub(r"(:\s*)(True|False|None)\b",
+                            lambda m: m.group(1) + {"True": "true", "False": "false",
+                                                    "None": "null"}[m.group(2)], text),
+    ),
+    (
+        "converted single-quoted strings to double-quoted",
+        lambda text: re.sub(r"'([^'\"]*)'", r'"\1"', text),
+    ),
+)
+
+
+class RelayJsonError(ValueError):
+    """Unparseable even after repairs — carries a pointed-at fragment."""
+
+
+def loads(payload: str) -> tuple[Any, list[str]]:
+    """Parse *payload*, repairing common model mistakes.
+
+    Returns (data, repairs) where *repairs* names what had to be fixed — empty
+    when the input was already valid. Raises RelayJsonError with a caret at the
+    offending character when nothing works."""
+    text = str(payload or "").strip()
+    if not text:
+        raise RelayJsonError("empty payload")
+    try:
+        return json.loads(text), []
+    except json.JSONDecodeError as exc:
+        # Bound to a plain name: Python deletes the `as` target at the end of
+        # the except block, so referring to it further down would raise
+        # UnboundLocalError instead of reporting the parse failure.
+        first_error = exc
+
+    applied: list[str] = []
+    candidate = text
+    for name, repair in _REPAIRS:
+        repaired = repair(candidate)
+        if repaired == candidate:
+            continue
+        candidate = repaired
+        applied.append(name)
+        try:
+            return json.loads(candidate), applied
+        except json.JSONDecodeError:
+            continue  # keep going: a payload can be broken in more than one way
+
+    raise RelayJsonError(point_at(text, first_error))
+
+
+def point_at(text: str, error: json.JSONDecodeError, width: int = 28) -> str:
+    """Show the failure where it happened.
+
+    A model can act on "you wrote `false\"}` here"; it cannot act on
+    "Expecting ',' delimiter: line 1 column 75"."""
+    position = max(0, min(len(text), getattr(error, "pos", 0)))
+    start = max(0, position - width)
+    end = min(len(text), position + width)
+    fragment = text[start:end]
+    caret = " " * (position - start) + "^"
+    lead = "…" if start > 0 else ""
+    tail = "…" if end < len(text) else ""
+    return (
+        f"{error.msg} at character {position}:\n"
+        f"  {lead}{fragment}{tail}\n"
+        f"  {' ' * len(lead)}{caret}"
+    )
+
+
+def repair_note(repairs: list[str]) -> str:
+    """The line appended to a tool result when repairs were needed.
+
+    Phrased as feedback rather than an apology: the model is the author of the
+    broken JSON and is the one who can stop sending it."""
+    if not repairs:
+        return ""
+    return (
+        " (note: your JSON was malformed — I " + "; ".join(repairs)
+        + " and ran it anyway. Send valid single-line JSON next time.)"
+    )

bolt_assistant/server.py

@@ -0,0 +1,186 @@
+"""The desk API: conversation (with the command relay) and text-to-speech.
+
+    text  -> POST /desk/converse_stream  (NDJSON, sentence by sentence)
+             or /desk/converse
+    [server may relay a `command`; the answer goes to /desk/tool_result]
+    reply <- final text
+    text  -> POST /desk/tts  -> raw 16 kHz mono PCM16, streamed
+
+Same protocol and conventions as Bolt Console's server_client and Bolt-Pet's
+tts, which this is adapted from.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Callable, Iterable, Iterator, NamedTuple, Optional
+
+import requests
+
+from . import config
+
+_MAX_RELAY_HOPS = 16
+
+
+class ServerError(Exception):
+    pass
+
+
+class Reply(NamedTuple):
+    text: str
+    streamed: bool = False  # already handed over sentence by sentence
+    voice_id: str = ""      # the server's speak_as pick, if any
+
+
+def _headers() -> dict:
+    return {"X-Desk-Api-Key": config.API_KEY}
+
+
+def check_health(timeout: float = 8.0) -> dict:
+    response = requests.get(f"{config.SERVER_URL}/desk/health", headers=_headers(), timeout=timeout)
+    response.raise_for_status()
+    return response.json()
+
+
+def _reply_from(payload: dict, streamed: bool = False) -> Reply:
+    if payload.get("type") == "reply":
+        return Reply(str(payload.get("text") or ""), streamed, str(payload.get("voice_id") or ""))
+    raise ServerError(str(payload.get("error") or "unknown server response"))
+
+
+def _relay(payload: dict, on_command, on_say) -> dict:
+    for _ in range(_MAX_RELAY_HOPS):
+        if payload.get("type") != "command":
+            break
+        holding = str(payload.get("say") or "").strip()
+        if holding and on_say is not None:
+            try:
+                on_say(holding)
+            except Exception:
+                pass
+        output = on_command(str(payload.get("command") or ""))
+        try:
+            payload = requests.post(
+                f"{config.SERVER_URL}/desk/tool_result",
+                json={"session_id": config.SESSION_ID, "token": payload.get("token"), "output": output},
+                headers=_headers(), timeout=180,
+            ).json()
+        except Exception as exc:
+            raise ServerError(f"couldn't reach the server during tool relay: {exc}") from exc
+    return payload
+
+
+def converse(text: str, on_command: Callable[[str], str],
+             on_say: Optional[Callable[[str], None]] = None, timeout: float = 120.0) -> Reply:
+    try:
+        payload = requests.post(
+            f"{config.SERVER_URL}/desk/converse",
+            json={"session_id": config.SESSION_ID, "text": text},
+            headers=_headers(), timeout=timeout,
+        ).json()
+    except Exception as exc:
+        raise ServerError(f"couldn't reach the server: {exc}") from exc
+    return _reply_from(_relay(payload, on_command, on_say))
+
+
+def converse_stream(text: str, on_say: Callable[[str], None], on_command: Callable[[str], str],
+                    timeout: float = 180.0) -> Reply:
+    """Each sentence to *on_say* as it's generated, so speaking starts after
+    the first sentence, not the whole answer. Raises ServerError only if
+    nothing was said yet (the caller then uses converse())."""
+    said = []
+    try:
+        response = requests.post(
+            f"{config.SERVER_URL}/desk/converse_stream",
+            json={"session_id": config.SESSION_ID, "text": text},
+            headers=_headers(), timeout=timeout, stream=True,
+        )
+        response.raise_for_status()
+        for raw in response.iter_lines(decode_unicode=True):
+            if not raw:
+                continue
+            try:
+                event = json.loads(raw)
+            except (TypeError, ValueError):
+                continue
+            kind = str(event.get("type") or "")
+            if kind == "say":
+                line = str(event.get("text") or "").strip()
+                if line:
+                    said.append(line)
+                    on_say(line)
+            elif kind == "command":
+                response.close()
+                return _reply_from(_relay(event, on_command, on_say))
+            elif kind == "reply":
+                if event.get("already_spoken"):
+                    return _reply_from(event, streamed=True)
+                return unsaid(_reply_from(event), said)
+            elif kind == "error":
+                raise ServerError(str(event.get("error") or "stream failed"))
+    except ServerError:
+        raise
+    except Exception as exc:
+        if said:
+            return Reply("", streamed=True)  # never repeat half an answer out loud
+        raise ServerError(f"streaming failed: {exc}") from exc
+    raise ServerError("stream ended without a reply")
+
+
+def unsaid(reply: Reply, said: list) -> Reply:
+    """The part of *reply* not already spoken as streamed sentences — the
+    server can stream the sentences and then send the whole text again
+    without `already_spoken` (compared ignoring whitespace)."""
+    if not said:
+        return reply
+    target = len("".join("".join(said).split()))
+    seen, cut = 0, None
+    for index, char in enumerate(reply.text):
+        if not char.isspace():
+            seen += 1
+        if seen == target:
+            cut = index + 1
+            break
+    if cut is None or "".join(reply.text[:cut].split()) != "".join("".join(said).split()):
+        return reply
+    rest = reply.text[cut:].strip()
+    return Reply(rest, streamed=not rest, voice_id=reply.voice_id)
+
+
+# ── text to speech ──────────────────────────────────────────────────────────
+
+class TtsError(Exception):
+    pass
+
+
+def tts_stream(text: str, voice_id: str = "", chunk_bytes: int = 4096) -> Iterator[bytes]:
+    """PCM16 bytes as the server synthesises them (whole samples only).
+    Raises TtsError before yielding anything if the request fails."""
+    voice = (voice_id or "").strip() or config.VOICE_ID
+    if not voice:
+        raise TtsError("ELEVENLABS_VOICE_ID isn't set, so the server doesn't know which voice to use")
+    try:
+        response = requests.post(
+            f"{config.SERVER_URL}/desk/tts", headers=_headers(),
+            json={"session_id": config.SESSION_ID, "text": text, "voice_id": voice},
+            timeout=60, stream=True,
+        )
+        response.raise_for_status()
+    except Exception as exc:
+        raise TtsError(f"text-to-speech request failed: {exc}") from exc
+    return whole_samples(response.iter_content(chunk_size=chunk_bytes))
+
+
+def whole_samples(chunks: Iterable[bytes]) -> Iterator[bytes]:
+    """HTTP chunks split wherever they like, including inside a 16-bit
+    sample. Carry the odd byte forward, or everything after the first split
+    plays as static (a Bolt-Pet lesson)."""
+    carry = b""
+    for chunk in chunks:
+        if not chunk:
+            continue
+        data = carry + chunk
+        usable = len(data) - len(data) % 2
+        carry = data[usable:]
+        if usable:
+            yield data[:usable]

bolt_assistant/speech_text.py

@@ -0,0 +1,173 @@
+"""Turn a server reply into something worth *hearing*.
+
+The server's persona writes for a chat window: markdown emphasis, bullet
+lists, emoji, bare URLs. A TTS voice reads those literally ("asterisk
+asterisk OS colon", "https colon slash slash..."), so everything spoken goes
+through for_speech() first. Pure string logic, no Qt/audio imports — cheap to
+unit test (see tests/test_speech_text.py).
+
+for_display() is the lighter sibling used for the speech bubble: it drops the
+markdown *syntax* but keeps emoji and punctuation, since those render fine.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Pictographs, symbols, flags, dingbats, arrows, box drawing, variation
+# selectors, ZWJ — anything a voice would either skip or read as a name
+# ("black right-pointing triangle").
+_EMOJI = re.compile(
+    "["
+    "\U0001F000-\U0001FAFF"  # emoji / pictographs / symbols blocks
+    "\U00002190-\U000021FF"  # arrows
+    "\U00002300-\U000023FF"  # misc technical (⌘ ⏱ …)
+    "\U000025A0-\U000027BF"  # geometric shapes, misc symbols, dingbats
+    "\U00002B00-\U00002BFF"  # extra arrows / shapes
+    "\U0000FE00-\U0000FE0F"  # variation selectors
+    "\U0001F1E6-\U0001F1FF"  # regional indicators (flags)
+    "\U0000200D"             # zero-width joiner
+    "]+",
+    flags=re.UNICODE,
+)
+
+# Characters that are markup or decoration rather than speech. Kept out of
+# the spoken text entirely; ordinary punctuation (. , ! ? ; : ' " ( ) -) is
+# preserved because it shapes prosody.
+_UNSPEAKABLE = re.compile(r"[*_#`~^|<>\\{}\[\]/=+@©®™•·–—]+")
+
+_FENCED_CODE = re.compile(r"```.*?```", re.DOTALL)
+_INLINE_CODE = re.compile(r"`([^`]*)`")
+_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
+_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]*\)")
+_URL = re.compile(r"\b(?:https?://|www\.)\S+")
+_HEADING = re.compile(r"^\s{0,3}#{1,6}\s*", re.MULTILINE)
+_BLOCKQUOTE = re.compile(r"^\s{0,3}>\s?", re.MULTILINE)
+_RULE = re.compile(r"^\s*([-*_=])(?:\s*\1){2,}\s*$", re.MULTILINE)
+_BULLET = re.compile(r"^\s*(?:[-*+•·]|\d+[.)])\s+", re.MULTILINE)
+_EMPHASIS = re.compile(r"(\*{1,3}|_{1,3})(\S(?:.*?\S)?)\1", re.DOTALL)
+_TABLE_PIPE = re.compile(r"[ \t]*\|[ \t]*")
+
+# Symbols worth saying out loud rather than dropping — a bare "&" read as
+# nothing turns "R&D" into "RD". Split in two because the non-ASCII ones sit
+# inside the arrow/symbol blocks _EMOJI strips, so they have to be worded
+# before that pass; the ASCII ones must wait until *after* markdown parsing
+# (an "=" turned into " equals " would stop a "====" rule matching _RULE).
+_PRE_SPOKEN_SYMBOLS = {
+    "→": " to ",
+    "×": " by ",
+    "°": " degrees ",
+    "≈": " about ",
+}
+_SPOKEN_SYMBOLS = {
+    "&": " and ",
+    "%": " percent ",
+    "@": " at ",
+    "+": " plus ",
+    "=": " equals ",
+}
+
+_MULTI_SPACE = re.compile(r"[ \t]+")
+_MULTI_PUNCT = re.compile(r"(?:\s*\.){2,}")
+
+
+def _strip_markdown(text: str, *, keep_emoji: bool) -> str:
+    text = _FENCED_CODE.sub(" (code) ", text)
+    text = _INLINE_CODE.sub(r"\1", text)
+    text = _MD_IMAGE.sub(r"\1", text)
+    text = _MD_LINK.sub(r"\1", text)
+    text = _RULE.sub("", text)
+    text = _HEADING.sub("", text)
+    text = _BLOCKQUOTE.sub("", text)
+    text = _EMPHASIS.sub(r"\2", text)
+    if not keep_emoji:
+        text = _EMOJI.sub(" ", text)
+    return text
+
+
+def _bullets_to_sentences(text: str) -> str:
+    """A read-aloud list needs pauses where the eye would see line breaks,
+    otherwise "OS Linux Uptime 1 day CPU load moderate" runs together."""
+    lines = [_BULLET.sub("", line).strip() for line in text.splitlines()]
+    lines = [line for line in lines if line]
+    if len(lines) < 2:
+        return lines[0] if lines else ""
+    # A bullet like "OS: Linux" ends mid-thought — give the voice a full stop
+    # so consecutive items don't slur into one run-on sentence.
+    return " ".join(line if line[-1] in ".!?:,;" else line + "." for line in lines)
+
+
+# ElevenLabs v3 delivery tags — "[laughing]", "[whispering]", "[sighs]". They
+# are instructions to the *dialogue* model (see dialogue.py), and only inside a
+# dialoguectl scene. Left in an ordinary reply they reach eleven_flash_v2,
+# which has no idea what they mean and simply reads the word: observed live
+# 2026-07-31, the pet announcing "laughing That one came through clean".
+#
+# Matched narrowly — a bracketed adverb/gerund, or one of the noise words that
+# aren't either — so real bracketed text ("[1]", "[see the docs]") survives.
+_AUDIO_TAG_RE = re.compile(
+    r"\[\s*(?:"
+    r"[a-z]+(?:ly|ing)"
+    r"|laughs?|chuckles?|sighs?|exhales?|inhales?|gulps?|pause|beat"
+    r"|clears throat|shouts?|whispers?|cries|sings?|gasps?|snorts?"
+    r"|sarcastic|excited|curious|nervous|angry|sad|happy|deadpan|flat"
+    r")\s*\]",
+    re.IGNORECASE,
+)
+
+
+def strip_audio_tags(text: str) -> str:
+    """Remove v3 delivery tags from text destined for the ordinary voice."""
+    return _AUDIO_TAG_RE.sub(" ", str(text or ""))
+
+
+def for_speech(text: str) -> str:
+    """Plain prose for the TTS engine: no markdown, no emoji, no bare URLs,
+    no stray symbols that would be read out character by character."""
+    text = strip_audio_tags((text or "").strip())
+    if not text:
+        return ""
+    for symbol, spoken in _PRE_SPOKEN_SYMBOLS.items():
+        text = text.replace(symbol, spoken)
+    text = _strip_markdown(text, keep_emoji=False)
+    text = _URL.sub(" link ", text)
+    text = _TABLE_PIPE.sub(", ", text)
+    text = _bullets_to_sentences(text)
+    for symbol, spoken in _SPOKEN_SYMBOLS.items():
+        text = text.replace(symbol, spoken)
+    text = _UNSPEAKABLE.sub(" ", text)
+    text = _MULTI_PUNCT.sub(".", text)
+    text = _MULTI_SPACE.sub(" ", text)
+    text = re.sub(r"\s+([.,!?;:])", r"\1", text)
+    return text.strip()
+
+
+def is_question(text: str) -> bool:
+    """True if the reply *ends* by asking the user something — the cue for
+    the pet to keep listening instead of making you say the wake word again.
+
+    Deliberately only looks at the end. A reply that asks something in
+    passing ("What time is it? It's 7:15.") isn't waiting on an answer,
+    whereas one that finishes on a question mark is. The test runs on the
+    spoken form, so a '?' that only exists inside a stripped code block or a
+    URL doesn't count, and trailing decoration (emoji, quotes, brackets) is
+    peeled off first so "Ready to go? 🚀" still reads as a question."""
+    spoken = for_speech(text)
+    while spoken and not (spoken[-1].isalnum() or spoken[-1] == "?"):
+        spoken = spoken[:-1]
+    return spoken.endswith("?")
+
+
+def for_display(text: str) -> str:
+    """What the speech bubble shows: markdown syntax removed (the bubble
+    can't render it) but emoji and layout-ish punctuation left alone.
+
+    Delivery tags go too — the voice no longer says them, so showing them
+    would caption a laugh nobody heard."""
+    text = strip_audio_tags((text or "").strip())
+    if not text:
+        return ""
+    text = _strip_markdown(text, keep_emoji=True)
+    lines = [_BULLET.sub("• ", line).strip() for line in text.splitlines()]
+    text = " ".join(line for line in lines if line)
+    return _MULTI_SPACE.sub(" ", text).strip()

bolt_assistant/stt.py

@@ -0,0 +1,152 @@
+"""Speech-to-text for one utterance, through the Bolt server's /desk/stt
+websocket relay (to Deepgram) — the route Bolt-Pet and the console's mic use,
+authenticated with the desk API key; no Deepgram account here.
+
+    t = Transcription(connect)
+    t.add(frames)        # as the utterance is spoken (streamed live)
+    text = t.finish(audio)   # when it ends; `audio` is the whole utterance
+    t.cancel()           # it was too short to matter
+
+Live when possible, so the text is ready the moment you stop talking. If the
+socket never opened or died, finish() opens a fresh one and sends `audio`
+in one go, from an empty transcript (so a half-live session can't return
+words twice). A healthy socket is never re-sent — that would pay twice.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import urllib.parse
+from typing import Callable, List, Optional
+
+from . import config
+
+
+class SttError(Exception):
+    pass
+
+
+def connect(sample_rate: int = config.SAMPLE_RATE):
+    import websocket  # websocket-client
+
+    query = urllib.parse.urlencode({"session_id": config.SESSION_ID, "encoding": "linear16",
+                                    "sample_rate": sample_rate})
+    return websocket.create_connection(
+        config.SERVER_URL.replace("http", "ws", 1) + "/desk/stt?" + query,
+        header=[f"X-Desk-Api-Key: {config.API_KEY}"], timeout=10)
+
+
+def parse_result(message) -> tuple:
+    """(text, is_final) from a Deepgram results frame; ("", False) otherwise."""
+    try:
+        if isinstance(message, bytes):
+            message = message.decode("utf-8", "ignore")
+        data = json.loads(message)
+    except (TypeError, ValueError):
+        return "", False
+    if not isinstance(data, dict) or data.get("type") not in (None, "Results"):
+        return "", False
+    alternatives = (data.get("channel") or {}).get("alternatives") or []
+    if not alternatives:
+        return "", False
+    text = str((alternatives[0] or {}).get("transcript") or "").strip()
+    return text, bool(data.get("is_final") or data.get("speech_final"))
+
+
+class Transcription:
+    def __init__(self, connect_fn: Callable = connect):
+        self._connect = connect_fn
+        self._finals: List[str] = []
+        self._interim = ""
+        self._lock = threading.Lock()
+        self._socket = None
+        self._ok = False
+        self._reader: Optional[threading.Thread] = None
+        self._open()
+
+    def _open(self) -> None:
+        try:
+            self._socket = self._connect(config.SAMPLE_RATE)
+            self._ok = True
+        except Exception:
+            self._socket, self._ok = None, False
+            return
+        self._reader = threading.Thread(target=self._read, name="stt-reader", daemon=True)
+        self._reader.start()
+
+    def add(self, frames: List[bytes]) -> None:
+        for frame in frames:
+            self._send(frame)
+
+    def _send(self, data: bytes) -> None:
+        if self._ok:
+            try:
+                self._socket.send_binary(data)
+            except Exception:
+                self._ok = False
+
+    def finish(self, audio: bytes) -> str:
+        """The transcript. Raises SttError only if even the fallback couldn't
+        reach the server."""
+        if self._ok:
+            self._flush()
+            return self._text()
+        self._close()
+        if self._reader is not None:
+            self._reader.join(timeout=1)
+        with self._lock:
+            self._finals, self._interim = [], ""
+        self._open()
+        if not self._ok:
+            raise SttError("couldn't reach the transcription server")
+        step = config.FRAME_SAMPLES * 2 * 8
+        for offset in range(0, len(audio), step):
+            self._send(audio[offset:offset + step])
+        self._flush()
+        return self._text()
+
+    def cancel(self) -> None:
+        self._ok = False
+        self._close()
+
+    def _flush(self) -> None:
+        """Deepgram sends its last results after CloseStream; wait for them,
+        bounded, so a hung socket can't hold up the reply."""
+        try:
+            self._socket.send(json.dumps({"type": "CloseStream"}))
+        except Exception:
+            pass
+        if self._reader is not None:
+            self._reader.join(timeout=4)
+        self._close()
+
+    def _close(self) -> None:
+        socket, self._socket = self._socket, None
+        if socket is not None:
+            try:
+                socket.close()
+            except Exception:
+                pass
+
+    def _read(self) -> None:
+        socket = self._socket
+        while socket is not None:
+            try:
+                message = socket.recv()
+            except Exception:
+                break
+            if not message:
+                break
+            text, is_final = parse_result(message)
+            with self._lock:
+                if is_final:
+                    if text:
+                        self._finals.append(text)
+                    self._interim = ""
+                elif text:
+                    self._interim = text
+
+    def _text(self) -> str:
+        with self._lock:
+            return " ".join(self._finals + ([self._interim] if self._interim else [])).strip()

bolt_assistant/tools.py

@@ -0,0 +1,93 @@
+"""What happens to a command the server relays back, when there's no screen
+to approve it on.
+
+ASSISTANT_COMMANDS decides:
+
+    off    nothing runs
+    read   (default) filectl list/read only: "what's in my Downloads?" works,
+           nothing on the machine can change
+    all    everything, shell commands and file writes included, like
+           Bolt-Pet. Anyone within earshot of the mic can ask for anything.
+
+A refusal goes back to the model as the command's output, phrased so it
+tells you rather than retrying. Pet-only pseudo-commands (petctl,
+dialoguectl) are explained, never run.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Callable
+
+from . import config, file_ops
+
+_PET = ("petctl", "bolt-pet", "dialoguectl")
+_LABEL = re.compile(r"^(?:command|cmd)\s*:\s*", re.IGNORECASE)
+
+
+def run_shell(command: str, timeout: int = None) -> str:
+    timeout = timeout or config.COMMAND_TIMEOUT_SECONDS
+    try:
+        done = subprocess.run(
+            command, shell=True, capture_output=True, text=True, errors="replace",
+            stdin=subprocess.DEVNULL, timeout=timeout, cwd=str(Path.home()),
+            creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
+        )
+        return f"[exit {done.returncode}]\n{(done.stdout or '') + (done.stderr or '')}"[:6000]
+    except subprocess.TimeoutExpired:
+        return f"[command timed out after {timeout}s]"
+    except Exception as exc:
+        return f"[command failed: {exc}]"
+
+
+def refusal(what: str, mode: str) -> str:
+    return (f"[assistant] Not run: this is Bolt Assistant, a voice-only client, and it's set to "
+            f"ASSISTANT_COMMANDS={mode}, which doesn't allow {what}. Tell the user in a sentence "
+            "that you can't do that from here (Bolt Console can); don't retry.")
+
+
+class Tools:
+    def __init__(self, mode: str = None, log: Callable[[str], None] = print,
+                 shell: Callable[[str], str] = run_shell):
+        mode = mode or config.COMMANDS
+        self.mode = mode if mode in ("off", "read", "all") else "read"  # unknown: the safe default
+        self.log = log
+        self.shell = shell
+        self.calls = 0
+
+    def __call__(self, command: str) -> str:
+        self.calls += 1
+        command = _LABEL.sub("", (command or "").strip(), count=1).strip().strip("`").strip()
+        if not command:
+            return "[assistant] empty command"
+        first = command.split(None, 1)[0].lower()
+        if first in _PET:
+            self.log(f"  (declined {first}: no pet body here)")
+            return (f"[assistant] {first} isn't available: this is Bolt Assistant, voice only, "
+                    "with no pet body or screen.")
+        try:
+            action = file_ops.parse(command)
+        except file_ops.FileOpError as exc:
+            return f"[filectl] {exc}"
+        if action is not None:
+            return self._file(action)
+        if self.mode != "all":
+            self.log(f"  (declined shell command: {command[:80]})")
+            return refusal("shell commands", self.mode)
+        self.log(f"  $ {command[:120]}")
+        return self.shell(command)
+
+    def _file(self, action: dict) -> str:
+        kind = action["action"]
+        allowed = {"off": set(), "read": {"help", "list", "read"},
+                   "all": {"help", "list", "read", "write", "edit"}}[self.mode]
+        if kind not in allowed:
+            self.log(f"  (declined file {kind}: {action.get('path', '')})")
+            return refusal(f"file {kind}s" if kind != "help" else "file access", self.mode)
+        self.log(f"  file {kind}: {action.get('path', '')}")
+        try:
+            return file_ops.execute(action)
+        except file_ops.FileOpError as exc:
+            return f"[filectl] {exc}"

bolt_assistant/wake.py

@@ -0,0 +1,50 @@
+"""Finding the wake phrase in a transcript, and what was said after it.
+
+    >>> find_wake("Hey Bolt, what's the weather?", WAKE)
+    (True, "what's the weather?")
+    >>> find_wake("tighten that bolt", WAKE)
+    (False, '')
+
+Rules, all aimed at not waking up on ordinary conversation:
+
+- The phrase has to **open** what was said. Only filler may come before it
+  ("um, hey Bolt", "okay so Bolt"), at most two words of it, so "tighten
+  that bolt" and "the bolt on the door" never count.
+- Whole words only: "boltage" is not "bolt".
+- Longest phrase wins, so "hey thunderbolt" isn't read as "hey" + garbage.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Iterable, Tuple
+
+_WORD = re.compile(r"[a-z0-9']+")
+_LEAD_SLACK = 2  # filler words allowed before the wake phrase
+_FILLER = {"um", "uh", "erm", "er", "so", "oh", "ok", "okay", "well", "and", "right",
+           "alright", "yeah", "hmm", "hm", "ah", "hi", "hey", "yo"}
+
+
+def _words(text: str) -> list:
+    """Lower-case words with their character spans in the original text."""
+    return [(m.group(0), m.start(), m.end()) for m in _WORD.finditer(text.lower())]
+
+
+def find_wake(transcript: str, phrases: Iterable[str]) -> Tuple[bool, str]:
+    """(heard, remainder): whether a wake phrase opens *transcript*, and the
+    request that followed it in the same breath ("" if it was just the
+    phrase)."""
+    words = _words(transcript or "")
+    if not words:
+        return False, ""
+    candidates = sorted({tuple(_WORD.findall(p.lower())) for p in phrases if p.strip()},
+                        key=len, reverse=True)
+    for start in range(min(_LEAD_SLACK + 1, len(words))):
+        if any(word not in _FILLER for word, _, _ in words[:start]):
+            break  # a real word came first: this isn't addressed to Bolt
+        for phrase in candidates:
+            span = words[start:start + len(phrase)]
+            if len(span) == len(phrase) and all(w == p for (w, _, _), p in zip(span, phrase)):
+                rest = transcript[span[-1][2]:]
+                return True, rest.lstrip(" \t,.;:!?-—").strip()
+    return False, ""