TheMajesticNetwork/Bolt-Pet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

feat: Enhance local command handling and introduce local intents

Browse Source 
- Refactor `run_local_command` to manage subprocesses more effectively, ensuring child processes are terminated on timeout.
- Introduce `_terminate` function to handle process group termination and capture output.
- Implement `_command_output` to format command results with a character limit.
- Add local intent recognition in `intents.py` to handle commands like "stop", "go to sleep", and "come here" without server interaction.
- Normalize user input to match local intents while stripping filler words.
- Update tests to cover new local intent functionality and ensure proper command handling.
- Enhance speech processing to handle abbreviations and improve spoken output clarity.
This commit is contained in:
Aaron Franks
2026-08-05 18:31:02 -06:00
parent 8d4751d80f
commit 3ee67cb4d6
14 changed files with 1107 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bolt_pet/audio/mic.py
+36
View File
@@ -37,6 +37,42 @@ def rms(frame: np.ndarray) -> float:
return float(np.sqrt(np.mean(frame.astype(np.float64) ** 2)))
def flush(stream, max_seconds: float = 10.0, sample_rate: int = config.SAMPLE_RATE) -> int:
"""Throw away whatever is already sitting in the mic's buffer. Returns the
number of frames dropped.
PortAudio keeps capturing into a ring buffer while nothing is reading it, so
audio recorded during a long blocking stretch is still queued when the next
read happens. That matters exactly once: at the end of a reply the pet is
about to listen for an answer, and the last fraction of a second of its own
TTS is in that buffer. It's above the VAD threshold, so `record_utterance`
treats it as the start of your answer, Deepgram transcribes it, and Bolt is
handed his own sentence as if you had said it. With barge-in on, the
detector was draining the stream during playback and the window is small;
with `BARGE_IN=false` nothing drains it at all.
Only safe where the buffer is known to hold *nothing you said* — never
before a wake-triggered recording, where the rest of "thunderbolt, what
time is it" is legitimately queued and dropping it clips the request.
*max_seconds* bounds a single call so this can't chase a stream that's
filling as fast as it's read. Best-effort: a fake stream in tests has no
`read_available` and this is a no-op, which is the correct behaviour for
one."""
try:
available = int(getattr(stream, "read_available", 0) or 0)
except (TypeError, ValueError):
return 0
if available <= 0:
return 0
frames = min(available, int(max_seconds * sample_rate))
try:
stream.read(frames)
except Exception:
return 0 # a mid-flush device error is the reader's problem, not ours
return frames
def record_utterance(
stream: AudioStream,
should_continue=lambda: True,
bolt_pet/config.py
+16
View File
@@ -123,6 +123,14 @@ FOLLOW_UP_LISTEN = os.environ.get("FOLLOW_UP_LISTEN", "true").lower() in ("1", "
FOLLOW_UP_MAX_TURNS = int(os.environ.get("FOLLOW_UP_MAX_TURNS", "10"))
FOLLOW_UP_GRACE_SECONDS = float(os.environ.get("FOLLOW_UP_GRACE_SECONDS", "7"))
# ── local intents ───────────────────────────────────────────────────────────
# A short, closed list of utterances the pet answers itself instead of paying a
# server round trip for: "stop", "come here", "go to sleep", "say that again",
# "use your normal voice". Matched whole and exact (see intents.py), never
# during a follow-up turn, so a real request is never swallowed. Turn it off to
# route absolutely everything through Bolt.
LOCAL_INTENTS = os.environ.get("LOCAL_INTENTS", "true").lower() in ("1", "true", "yes", "on")
COMMAND_TIMEOUT_SECONDS = int(os.environ.get("COMMAND_TIMEOUT_SECONDS", "30"))
# ── sudo password prompts ───────────────────────────────────────────────────
@@ -221,6 +229,14 @@ NOTIFICATION_BRIDGE = os.environ.get("NOTIFICATION_BRIDGE", "false").lower() in
# Regex matched against "<app>: <summary> <body>"; empty means "everything".
NOTIFICATION_FILTER = os.environ.get("NOTIFICATION_FILTER", "")
NOTIFICATION_MIN_INTERVAL_SECONDS = float(os.environ.get("NOTIFICATION_MIN_INTERVAL_SECONDS", "60"))
# Notifications arrive on the watcher thread and are forwarded from the
# heartbeat, which doesn't run while the pet is napping — so they queue. Both
# limits exist to stop an overnight backlog turning into a burst of round trips
# and a monologue at 8am: the queue is bounded (oldest dropped first) and
# anything staler than the age limit is discarded at drain time, because
# "Firefox finished downloading" is not news nine hours later.
NOTIFICATION_QUEUE_LIMIT = int(os.environ.get("NOTIFICATION_QUEUE_LIMIT", "20"))
NOTIFICATION_MAX_AGE_SECONDS = float(os.environ.get("NOTIFICATION_MAX_AGE_SECONDS", "900"))
# ── file delivery ────────────────────────────────────────────────────────
# The server's deliver_files tool (ai/desk_api.py in the main tmn-api repo)
bolt_pet/controller.py
+193 -27
View File
@@ -15,15 +15,16 @@ from __future__ import annotations
import threading
import time
from collections import deque
from typing import Optional
from PySide6.QtCore import QObject, Signal
from . import (
config, dialogue as dialogue_mod, file_delivery, file_ops,
history as history_mod, monitors as monitors_mod, notifications,
pet_actions, quiet, screen_context, screen_text, self_restart,
server_client, speech_text, updater,
history as history_mod, intents as intents_mod, monitors as monitors_mod,
notifications, pet_actions, quiet, screen_context, screen_text,
self_restart, server_client, speech_text, updater,
)
from . import __version__
from .audio import barge_in, mic, stt, tts, wake_word
@@ -98,7 +99,14 @@ class PetController(QObject):
self._notification_gate = notifications.NotificationGate(
config.NOTIFICATION_FILTER, config.NOTIFICATION_MIN_INTERVAL_SECONDS
)
self._pending_notifications: list[notifications.Notification] = []
# Bounded, and stamped on arrival: the drain only runs from the
# heartbeat, which doesn't run while napping, so this fills up
# overnight. maxlen drops the oldest rather than growing without limit,
# and the stamp lets the drain discard a backlog nobody wants read out
# at 8am (see _drain_notifications).
self._pending_notifications: deque[tuple[float, notifications.Notification]] = deque(
maxlen=max(1, config.NOTIFICATION_QUEUE_LIMIT)
)
self._notification_lock = threading.Lock()
# ── external controls (safe to call from the Qt/UI thread) ─────────
@@ -187,18 +195,47 @@ class PetController(QObject):
)
self.log.emit(f"Barge-in: {config.BARGE_IN_MODE} mode.")
with self._stream:
try:
health = server_client.check_health()
self.log.emit(f"Connected to server: {health}")
except Exception as exc:
self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.")
self._start_notification_bridge()
self._report_self_restart()
self._loop()
if self._notification_watcher is not None:
self._notification_watcher.stop()
self.finished.emit()
# Everything past here is in try/finally because `finished` is what
# ui/app.py waits on to quit the QThread and to run a pending
# os.execv. An exception escaping _loop used to skip it, leaving the
# thread wedged with the mic still open and no restart — so the failure
# mode of any bug below was "the pet goes deaf and the tray won't quit"
# rather than "one turn failed".
try:
with self._stream:
try:
health = server_client.check_health()
self.log.emit(f"Connected to server: {health}")
except Exception as exc:
self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.")
self._start_notification_bridge()
self._guarded(self._report_self_restart, "restart report")
self._loop()
except Exception as exc:
self.log.emit(f"Pipeline stopped unexpectedly: {exc!r}")
finally:
if self._notification_watcher is not None:
self._notification_watcher.stop()
self.finished.emit()
def _guarded(self, work, label: str) -> bool:
"""Run *work*, absorbing anything it raises.
The pipeline is one thread driving a state machine that raises on an
illegal transition (deliberately — see state.py), plus a dozen
best-effort subsystems that shell out, hit the network, or touch the
filesystem. Any one of them raising something unforeseen used to end the
whole session. Here, it costs a log line and a forced return to IDLE,
which is the only state it's always safe to resume from.
Returns True if *work* completed without raising."""
try:
work()
return True
except Exception as exc:
self.log.emit(f"Recovered from a {label} failure: {exc!r}")
self._state.force(PetState.IDLE)
return False
def _loop(self) -> None:
while self._running:
@@ -214,7 +251,7 @@ class PetController(QObject):
if not self._running:
return
continue
self._handle_conversation_turn()
self._guarded(self._handle_conversation_turn, "conversation turn")
def _wait_for_wake_or_click(self) -> bool:
"""True once either the wake phrase was heard or a click-to-talk
@@ -226,7 +263,12 @@ class PetController(QObject):
self._stream,
should_continue=should_continue,
threshold=self.wake_threshold, # callable: the tuner slider is live
on_tick=self._maybe_heartbeat,
# Guarded: on_tick is the one place control returns to us during a
# listen that can block for minutes, and everything it drives
# (update check, nap probe, notification forwarding) touches the
# network or shells out. Unguarded, any of them raising would unwind
# the listen loop and end the session.
on_tick=lambda: self._guarded(self._maybe_heartbeat, "heartbeat"),
on_score=self._observe_wake_score,
)
if not self._running:
@@ -275,6 +317,13 @@ class PetController(QObject):
self.log.emit(f"You: {text}")
self.history.add(history_mod.USER, text, time.time())
# "stop", "come here", "say that again" — answered here, without the
# round trip. Never on a follow-up turn: Bolt asked you something and
# the answer is his, even if it happens to look like a body command.
if not following_up and self._handle_local_intent(text):
self._state.transition(PetState.IDLE)
return
try:
# What's focused right now rides along, so "what's this error?"
# has a referent without you having to describe the window.
@@ -293,6 +342,60 @@ class PetController(QObject):
self._state.transition(PetState.IDLE)
self._maybe_self_restart()
def _handle_local_intent(self, text: str) -> bool:
"""Answer *text* locally if it's one of the closed set of body commands
in intents.py. Returns True if it was handled (no server call).
The effects live here rather than in intents.py for the same reason
pet_actions splits parse from describe: recognising the phrase is pure
and testable, doing the thing needs the controller's state, the tray's
nap override and a Qt signal to the window."""
if not config.LOCAL_INTENTS:
return False
intent = intents_mod.recognize(text)
if intent is None:
return False
self.log.emit(f"Local intent: {intent.name} (answered without the server)")
if intent.name == "stop":
# Nothing to say and nothing to do: silence is the acknowledgement.
# Also ends any follow-up chain — "never mind" means the
# conversation is over, not that we should keep the mic open.
self._follow_ups = 0
self._pending_follow_up = False
self._talk_now.clear()
return True
if intent.name == "repeat":
last = self.history.last(history_mod.PET)
if last is None:
self._speak("I haven't said anything yet.")
else:
# remember=False: replaying a line isn't a new turn. Appending it
# would make "say that again" twice over read back as a
# conversation where Bolt volunteered the same thing three times.
self._speak(last.text, remember=False)
return True
if intent.name == "voice_reset":
had_voice = bool(self._voice_id)
self.reset_voice()
self._speak(intent.speak if had_voice else "That is my normal voice.")
return True
action = intent.action
if action is not None:
if action.get("action") == "nap":
# Through set_napping, not just the signal, so a spoken "go to
# sleep" overrides the quiet-hours schedule exactly like the
# tray's Nap entry and `petctl nap` do — otherwise the next
# schedule check would undo it within ten seconds.
self.set_napping(bool(action["enabled"]))
self.action.emit(dict(action))
if intent.speak:
self._speak(intent.speak)
return True
def _with_context(self, text: str) -> str:
"""Everything the server gets alongside what you actually said: the
focused window title, and a one-line note about the screen layout so
@@ -319,9 +422,26 @@ class PetController(QObject):
self._pet_monitor = int(index)
def _handle_command(self, command: str) -> str:
"""Server-relayed command. `petctl ...` drives the pet's body and
`filectl ...` does local file read/write/edit — neither ever reaches
a shell; everything else is a real command, exactly as before (see
"""Server-relayed command, with the guarantee the relay depends on: this
always returns a string.
The server is blocked on `/desk/tool_result` while this runs. If it
raises instead of answering, the relay never posts, the turn dies
mid-flight, and the server sits out its own timeout on a conversation it
can't finish — the worst available failure mode, because it's silent on
both ends. Handing the exception back as command output instead means
Bolt can read what went wrong and say so, or try something else, inside
the same turn."""
try:
return self._dispatch_command(command)
except Exception as exc:
self.log.emit(f"Command handler failed: {exc!r}")
return f"[error] the pet couldn't run that: {exc}"
def _dispatch_command(self, command: str) -> str:
"""`petctl ...` drives the pet's body, `dialoguectl ...` plays a scene
and `filectl ...` does local file read/write/edit — none of them ever
reach a shell; everything else is a real command, exactly as before (see
the security notes in the README)."""
try:
action = pet_actions.parse(command)
@@ -568,14 +688,15 @@ class PetController(QObject):
elif not config.VOICE_STICKY:
self.reset_voice()
def _speak(self, text: str) -> None:
def _speak(self, text: str, remember: bool = True) -> None:
self._state.transition(PetState.TALKING)
# Bubble gets the markdown stripped but emoji kept (it can't render
# **bold** but draws emoji fine); tts.speak() does its own, stricter
# sanitizing for the voice.
self.said.emit(speech_text.for_display(text))
self.log.emit(f"Bolt: {text}")
self.history.add(history_mod.PET, text, time.time())
if remember:
self.history.add(history_mod.PET, text, time.time())
should_stop = None
if self._barge_in is not None:
@@ -608,6 +729,15 @@ class PetController(QObject):
f"Asked a question — listening for your answer "
f"({self._follow_ups}{'/' + str(cap) if cap > 0 else ''})."
)
# The tail of the reply we just played is still in the mic's ring
# buffer, and we're about to start recording with a VAD that will
# take it for the start of your answer — Bolt's own last words,
# transcribed and sent back to him as if you'd said them. Nothing you
# said can be in there: playback ran to completion, so if you had
# spoken, barge-in would have cut it and taken the other branch.
dropped = mic.flush(self._stream)
if dropped:
self.log.emit(f"Dropped {dropped} buffered frames of my own voice.")
self._pending_follow_up = True
self._talk_now.set()
@@ -686,16 +816,39 @@ class PetController(QObject):
def _queue_notification(self, notification: notifications.Notification) -> None:
"""Called on the watcher thread — just queue it; forwarding happens on
the pipeline thread where it can't collide with a live conversation."""
if not self._notification_gate.should_forward(notification, time.monotonic()):
now = time.monotonic()
if not self._notification_gate.should_forward(notification, now):
return
with self._notification_lock:
self._pending_notifications.append(notification)
if len(self._pending_notifications) == self._pending_notifications.maxlen:
# Say so rather than dropping in silence: a full queue means the
# bridge is matching more than the pet can plausibly speak, and
# the filter is what wants tightening.
self.log.emit("Notification queue full — dropping the oldest.")
self._pending_notifications.append((now, notification))
def _drain_notifications(self) -> None:
with self._notification_lock:
pending, self._pending_notifications = self._pending_notifications, []
for notification in pending:
pending = list(self._pending_notifications)
self._pending_notifications.clear()
now = time.monotonic()
max_age = config.NOTIFICATION_MAX_AGE_SECONDS
if max_age > 0:
fresh = [entry for entry in pending if now - entry[0] <= max_age]
if len(fresh) != len(pending):
self.log.emit(
f"Skipping {len(pending) - len(fresh)} notification(s) older than "
f"{int(max_age)}s."
)
pending = fresh
for index, (_stamped, notification) in enumerate(pending):
if not self._running or self._napping:
# Put back what we haven't forwarded — the old code swapped the
# queue out and then returned, silently dropping the remainder
# the moment a nap started mid-drain.
self._requeue_notifications(pending[index:])
return
self.log.emit(f"Notification: {notification.as_text()}")
self.history.add(history_mod.SYSTEM, notification.as_text(), time.time())
@@ -705,7 +858,11 @@ class PetController(QObject):
on_command=self._handle_command,
)
except server_client.ServerError as exc:
# Keep this one and everything behind it for the next heartbeat:
# the server being briefly down shouldn't silently eat the
# backlog. The age limit is what stops that retrying forever.
self.log.emit(f"Couldn't forward notification: {exc}")
self._requeue_notifications(pending[index:])
return
self._check_deliveries()
self._apply_voice(reply)
@@ -713,6 +870,15 @@ class PetController(QObject):
self._speak(reply.text)
self._state.transition(PetState.IDLE)
def _requeue_notifications(self, entries: list) -> None:
"""Push undelivered notifications back on the front, oldest first, so a
retry keeps their original order (and their original timestamps, so a
retry loop can't keep a stale one alive indefinitely)."""
if not entries:
return
with self._notification_lock:
self._pending_notifications.extendleft(reversed(entries))
# ── file delivery ────────────────────────────────────────────────────
def _check_deliveries(self) -> None:
bolt_pet/intents.py
+206
View File
@@ -0,0 +1,206 @@
"""Things you say to the pet that the server has no business answering.
"stop", "come here", "go to sleep", "say that again", "use your normal voice" —
none of these are questions for Bolt's brain. They're commands to the *body*,
and today every one of them costs a full turn: Deepgram, a `/desk/converse`
round trip, a model deciding to emit `petctl`, then ElevenLabs. Two to four
seconds and three network hops to make the pet walk left, and it only works at
all if the server's prompt happens to advertise the right verb — which is
exactly why `petctl voice reset` needs a block in the server's pet prompt (see
CLAUDE.md) or the model never emits it. Recognising the phrase here removes
both the latency and that coupling: "go back to your normal voice" works
whether or not the server was ever told the voice can be reset.
The whole design problem is **not stealing real requests**. Three rules keep
it honest:
1. **Whole-utterance, exact match after normalisation.** Never substring. So
"stop" is an intent and "stop the docker container" is a question for the
server — the distinction a substring match would destroy.
2. **The phrase table is closed and small.** Every entry is something with no
plausible reading as a request for Bolt to *do work*. Anything arguable
("no thanks", "nothing") is deliberately absent — see rule 3 for why a
wrong guess is expensive.
3. **Nothing is recognised mid-conversation.** The controller skips this
entirely on a follow-up turn: if Bolt just asked you something, your answer
belongs to him, and swallowing "never mind" locally would leave the server
holding a question it never got an answer to. Local intents are only ever
for turns *you* started.
Both sides of the comparison go through `normalize()` — the table is
canonicalised at import — so phrases can be written the way a person says them
("go back to your normal voice") without every variant having to be spelled
out. Filler is dropped from anywhere, not just the ends, because STT scatters
it ("hey bolt, could you please just stop now").
Pure classification, like pet_actions.parse: this module decides *what was
meant* and hands back an action in the same shape pet_actions produces, so
`controller.action` and `PetWindow.apply_action` need no new vocabulary. The
effects live in controller._handle_local_intent.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Optional
# Words with no bearing on any command in the table, dropped wherever they
# appear. Kept deliberately short: every entry here is a word that can't
# distinguish one of these phrases from another, and adding one that can is how
# two intents quietly collide (the builder below raises if that happens).
_FILLER = frozenset({
"the", "a", "an", "my", "your", "yours", "its", "to", "of", "and",
"please", "just", "that", "some", "bolt", "thunderbolt", "pet", "buddy",
})
# Dropped only from the front — the politeness/address ramp STT reliably
# prefixes. Not safe to drop mid-phrase (a bare "do" or "go" carries meaning
# elsewhere), which is why this is separate from _FILLER.
_LEADING_FILLER = frozenset({
"hey", "hi", "hello", "yo", "ok", "okay", "um", "uh", "er", "so",
"can", "could", "would", "will", "you", "i", "id", "like", "lets",
"let", "us", "do", "go", "then", "now",
})
_TRAILING_FILLER = frozenset({
"ok", "okay", "thanks", "thank", "you", "boy", "already", "now",
})
_KEEP = re.compile(r"[^a-z0-9 ]+")
def normalize(text: str) -> str:
"""Reduce an utterance to the bare command, or "" if nothing is left.
Lowercase, punctuation stripped (STT punctuates inconsistently), filler
dropped. Not a stemmer and deliberately not clever — its only job is to
make the same command spoken two ways land on the same string, without
ever turning one command into a different one."""
words = [word for word in _KEEP.sub(" ", (text or "").lower()).split()
if word not in _FILLER]
while words and words[0] in _LEADING_FILLER:
words.pop(0)
while words and words[-1] in _TRAILING_FILLER:
words.pop()
return " ".join(words)
@dataclass(frozen=True)
class Intent:
"""One recognised local command.
*action* is a pet_actions-shaped dict for the UI (or None when there's
nothing for the body to do); *speak* is what to say out loud, empty for the
intents where doing the thing silently *is* the acknowledgement — the pet
visibly moves, and a spoken confirmation would only make it slower. "stop"
in particular has to be silent: answering "okay!" when told to be quiet is
a comedy sketch, not a feature.
"""
name: str
action: Optional[dict] = None
speak: str = ""
# Intent -> (the Intent, the phrases that mean it, written as spoken).
_TABLE: tuple[tuple[Intent, tuple[str, ...]], ...] = (
(
Intent("stop"),
("stop", "stop talking", "stop it", "be quiet", "quiet", "shut up",
"hush", "never mind", "nevermind", "forget it", "cancel",
"cancel that", "drop it", "enough"),
),
(
Intent("nap", {"action": "nap", "enabled": True}, "Night."),
("go to sleep", "take a nap", "have a nap", "go to bed", "bedtime",
"goodnight", "good night", "get some rest"),
),
(
Intent("wake", {"action": "nap", "enabled": False}, "I'm up."),
("wake up", "get up", "rise and shine", "you're awake", "are you awake"),
),
(
Intent("come", {"action": "move", "anchor": "cursor"}),
("come here", "come to me", "come back", "over here", "follow me",
"follow my cursor"),
),
(
Intent("go_away", {"action": "move", "anchor": "bottom-right"}),
("go away", "move over", "move out of the way", "get out of the way",
"out of the way", "hide", "get lost", "shoo", "scram",
"go somewhere else"),
),
(
Intent("repeat"), # answered from history by the controller
("say that again", "say again", "repeat that", "repeat",
"what did you say", "what was that", "come again", "one more time",
"again", "sorry what"),
),
(
Intent("wander_on", {"action": "wander", "enabled": True}),
("go for a walk", "wander", "wander around", "walk around", "explore",
"stretch your legs", "roam"),
),
(
Intent("wander_off", {"action": "wander", "enabled": False}),
("stay still", "stay put", "stop moving", "stop wandering",
"don't move", "sit", "sit still", "stay", "settle down", "hold still"),
),
(
# Reachable from the server too (petctl voice reset), but only if its
# prompt mentions the verb. Recognising it here is what makes the
# phrase work regardless of what the server was told.
Intent("voice_reset", None, "Back to my own voice."),
("use your normal voice", "use your own voice", "your normal voice",
"go back to your normal voice", "be yourself", "be yourself again",
"stop doing that voice", "drop the voice", "talk normally",
"speak normally", "use your real voice"),
),
)
def _build() -> dict[str, Intent]:
"""Canonicalise the table, refusing to build an ambiguous one.
A phrase that normalises to "" would match an utterance of pure filler
("hey bolt"), and one that lands on the same string as a phrase from
another intent would silently bind to whichever was declared last. Both are
edit-time mistakes, so they fail at import rather than at 3am on a mic."""
table: dict[str, Intent] = {}
for intent, phrases in _TABLE:
for phrase in phrases:
key = normalize(phrase)
if not key:
raise ValueError(f"intent phrase {phrase!r} normalises to nothing")
existing = table.get(key)
if existing is not None and existing.name != intent.name:
raise ValueError(
f"phrase {phrase!r} ({key!r}) is claimed by both "
f"{existing.name} and {intent.name}"
)
table[key] = intent
return table
_BY_PHRASE = _build()
# Longest phrase in the table, in words. Anything longer can't match, so a real
# request skips normalisation entirely — this runs on every turn.
_MAX_WORDS = max(len(phrase.split()) for phrase in _BY_PHRASE)
def recognize(text: str) -> Optional[Intent]:
"""The intent *text* expresses, or None to send it to the server.
None is the safe answer and the common one: anything not matched verbatim
against the table belongs to Bolt."""
raw = (text or "").strip()
if not raw:
return None
# +6 words of slack for the filler about to be stripped ("hey bolt, could
# you please stop" is six words to reach a one-word command).
if len(raw.split()) > _MAX_WORDS + 6:
return None
intent = _BY_PHRASE.get(normalize(raw))
return intent
bolt_pet/server_client.py
+66 -7
View File
@@ -19,6 +19,8 @@ Kept dependency-free beyond `requests` so it's easy to unit test with mocks.
from __future__ import annotations
import os
import signal
import subprocess
from pathlib import Path
from typing import Callable, NamedTuple, Optional
@@ -29,6 +31,10 @@ from . import config, sudo_askpass
_MAX_RELAY_HOPS = 16
# Command output handed back up the relay is capped: it becomes part of the
# server's prompt, and a runaway `find /` would blow the context window.
_MAX_COMMAND_OUTPUT = 6000
class ServerError(Exception):
"""Raised when the server responds with an error payload or unreachable."""
@@ -74,17 +80,61 @@ def run_local_command(command: str, timeout: int = None) -> str:
timeout = timeout or config.SUDO_COMMAND_TIMEOUT_SECONDS
timeout = timeout or config.COMMAND_TIMEOUT_SECONDS
try:
completed = subprocess.run(
command, shell=True, capture_output=True, text=True,
timeout=timeout, cwd=str(Path.home()), env=env,
# start_new_session puts the shell in its own process group so a timeout
# can kill the whole tree. subprocess.run() would only SIGKILL the `sh`
# itself, leaving whatever it spawned (a build, a `tail -f`, an ffmpeg)
# running forever with no parent watching — one relayed command that
# hangs shouldn't leak a process for the rest of the session.
process = subprocess.Popen(
command, shell=True, cwd=str(Path.home()), env=env, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
start_new_session=(os.name == "posix"),
)
output = (completed.stdout or "") + (completed.stderr or "")
return f"[exit {completed.returncode}]\n{output}"[:6000]
except subprocess.TimeoutExpired:
return f"[command timed out after {timeout}s]"
except Exception as exc:
return f"[command failed: {exc}]"
try:
stdout, stderr = process.communicate(timeout=timeout)
return _command_output(f"[exit {process.returncode}]", stdout, stderr)
except subprocess.TimeoutExpired:
stdout, stderr = _terminate(process)
# Whatever it managed to print before it hung is the useful part — a
# bare "timed out" tells the model nothing it can act on, and the last
# line of output usually says exactly what it was stuck waiting for.
return _command_output(f"[command timed out after {timeout}s]", stdout, stderr)
except Exception as exc:
_terminate(process)
return f"[command failed: {exc}]"
def _terminate(process: subprocess.Popen) -> tuple[str, str]:
"""Kill a timed-out command's whole process group and collect what it wrote.
SIGTERM first so a shell script can clean up, SIGKILL a moment later for
anything that ignores it. The final drain is itself time-boxed: a
grandchild holding the pipe open must not turn a timeout into a hang."""
try:
if os.name == "posix":
group = os.getpgid(process.pid)
os.killpg(group, signal.SIGTERM)
try:
process.wait(timeout=2)
except subprocess.TimeoutExpired:
os.killpg(group, signal.SIGKILL)
else:
process.kill()
except (ProcessLookupError, PermissionError, OSError):
pass # already gone, or never had its own group
try:
return process.communicate(timeout=2)
except Exception:
return "", ""
def _command_output(header: str, stdout: Optional[str], stderr: Optional[str]) -> str:
body = (stdout or "") + (stderr or "")
return f"{header}\n{body}"[:_MAX_COMMAND_OUTPUT]
def converse(
text: str,
@@ -132,6 +182,15 @@ def converse(
voice_id=str(payload.get("voice_id") or ""),
voice_name=str(payload.get("voice_name") or ""),
)
if payload.get("type") == "command":
# Fell out of the loop still being handed commands. Worth its own
# message: "unknown server response" sent everyone looking at the
# payload shape, when what actually happened is a model that kept
# calling tools and never answered.
raise ServerError(
f"the server kept relaying commands past the {_MAX_RELAY_HOPS}-hop cap "
"without producing a reply"
)
raise ServerError(str(payload.get("error") or "unknown server response"))
bolt_pet/speech_text.py
+39 -10
View File
@@ -67,6 +67,27 @@ _SPOKEN_SYMBOLS = {
"=": " equals ",
}
# Abbreviations a voice spells out letter by letter ("eee gee") because the
# periods make them look like sentence boundaries. Written out instead — this
# has to run before _UNSPEAKABLE strips anything, and the trailing \.? keeps
# "etc" working with or without its period. Word-bounded so "vs" inside a
# filename is left alone.
_SPOKEN_ABBREVIATIONS = (
(re.compile(r"\be\.g\.?(?=\s|$)", re.IGNORECASE), "for example"),
(re.compile(r"\bi\.e\.?(?=\s|$)", re.IGNORECASE), "that is"),
(re.compile(r"\betc\.?(?=\s|$)", re.IGNORECASE), "and so on"),
(re.compile(r"\bvs\.?(?=\s|$)", re.IGNORECASE), "versus"),
(re.compile(r"\baka\b", re.IGNORECASE), "also known as"),
(re.compile(r"\bw/(?=\s)", re.IGNORECASE), "with"),
# "PR #42" -> "PR number 42"; a bare "#" is markup and _UNSPEAKABLE drops it.
(re.compile(r"#(?=\d)"), "number "),
# A long option's dashes are punctuation to the eye and syllables to the ear
# ("dash dash force"). Only the doubled form: a single hyphen has to survive
# for "bolt-pet" and "up-to-date", and requiring a word character after it
# keeps a "---" horizontal rule intact for _RULE to strip.
(re.compile(r"(?<!\w)--(?=\w)"), ""),
)
_MULTI_SPACE = re.compile(r"[ \t]+")
_MULTI_PUNCT = re.compile(r"(?:\s*\.){2,}")
@@ -105,6 +126,10 @@ def for_speech(text: str) -> str:
return ""
for symbol, spoken in _PRE_SPOKEN_SYMBOLS.items():
text = text.replace(symbol, spoken)
# Before the markdown pass, so "#42" still has its "#" to word and a real
# "## Heading" (no digit after the hashes) is left for _HEADING to strip.
for pattern, spoken in _SPOKEN_ABBREVIATIONS:
text = pattern.sub(spoken, text)
text = _strip_markdown(text, keep_emoji=False)
text = _URL.sub(" link ", text)
text = _TABLE_PIPE.sub(", ", text)
@@ -119,17 +144,21 @@ def for_speech(text: str) -> str:
def is_question(text: str) -> bool:
"""True if the spoken reply contains a question mark — the cue for the
pet to keep listening instead of making you say the wake word again.
"""True if the reply asks the user anything — the cue for the pet to keep
listening instead of making you say the wake word again.
The check runs on the spoken form, so a '?' that only exists inside a
stripped code block or a URL doesn't count, and trailing decoration
(emoji, quotes, brackets) is peeled off first so "Ready to go? 🚀"
still reads as a question."""
spoken = for_speech(text)
while spoken and not (spoken[-1].isalnum() or spoken[-1] in "?."):
spoken = spoken[:-1]
return "?" in spoken
Anywhere in the reply counts, not only the end. An earlier version required
a *trailing* '?' on the theory that "What time is it? It's 7:15." isn't
waiting on an answer, and that's true of that sentence but wrong far more
often: Bolt routinely asks first and then keeps talking ("Want me to fix
it? I'd start with the config."), and refusing to listen there is the case
that actually costs you a wake word. The cheap failure is the other
direction — an unwanted extra listen ends itself on `VAD_GRACE_SECONDS` of
silence, and `FOLLOW_UP_MAX_TURNS` caps the chain.
The test runs on the *spoken* form, so a '?' that only exists inside a
stripped code block, a URL, or a markdown link target doesn't count."""
return "?" in for_speech(text)
def for_display(text: str) -> str: