Add text-to-dialogue, self-restart capability, and misc updates

This commit is contained in:
2026-07-30 20:50:41 -06:00
parent 5b49670983
commit 96afc351ac
21 changed files with 1819 additions and 59 deletions
+96 -12
View File
@@ -5,11 +5,16 @@ desk_client/bolt_desk.py which shells out because it only targets Linux.
Falls back to pyttsx3 (offline, cross-platform: SAPI5 on Windows, NSSpeech
on macOS, espeak on Linux) if ElevenLabs isn't configured or the request
fails, so the pet can still talk with zero cloud config.
Every entry point takes an optional *voice_id* that overrides
`ELEVENLABS_VOICE_ID` for that call — that's how the server's `speak_as`
reply marker reaches the speakers (see controller._apply_voice). The offline
fallback has no such concept and always sounds like itself.
"""
from __future__ import annotations
from typing import Iterable, Iterator
from typing import Iterable, Iterator, Optional
import numpy as np
import requests
@@ -21,18 +26,40 @@ class TtsError(Exception):
pass
def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
def voice_for(voice_id: Optional[str] = None) -> str:
"""The voice this call should use: an override (server `speak_as`) if
given, else the configured default."""
return (voice_id or "").strip() or config.ELEVENLABS_VOICE_ID
def model_for(text: str, voice_id: Optional[str] = None) -> str:
"""Which ElevenLabs model to synthesize with.
The default (`eleven_flash_v2`) is English-only, and both things that
reach this branch mean the reply probably isn't English: a voice the
server picked mid-conversation is nearly always about a language or an
accent, and non-ASCII text can't be English at all. Rendering either one
through the English model gets you a mangled phonetic reading rather
than a failure, which is worse — so those go through the multilingual
model instead."""
if (voice_id or "").strip() or not text.isascii():
return config.ELEVENLABS_MULTILINGUAL_MODEL_ID
return config.ELEVENLABS_MODEL_ID
def synthesize_pcm(text: str, voice_id: Optional[str] = None) -> tuple[np.ndarray, int]:
"""Returns (pcm_int16_mono, sample_rate). Raises TtsError on failure —
callers should fall back to speak_offline() rather than treating this
as fatal."""
if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}",
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
json={"text": text, "model_id": model_for(text, voice_id)},
timeout=60,
)
response.raise_for_status()
@@ -44,20 +71,23 @@ def synthesize_pcm(text: str) -> tuple[np.ndarray, int]:
return pcm, config.TTS_SAMPLE_RATE
def stream_pcm(text: str, chunk_bytes: int = 4096) -> Iterator[np.ndarray]:
def stream_pcm(
text: str, chunk_bytes: int = 4096, voice_id: Optional[str] = None
) -> Iterator[np.ndarray]:
"""Same audio as synthesize_pcm(), but yielded as it arrives from
ElevenLabs' /stream endpoint so playback can start on the first chunk
(~300ms) instead of after the whole clip is synthesized. Raises TtsError
before yielding anything if the request itself fails, so callers can fall
back cleanly; a mid-stream failure just ends the generator."""
if not (config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID):
voice = voice_for(voice_id)
if not (config.ELEVENLABS_API_KEY and voice):
raise TtsError("ELEVENLABS_API_KEY / ELEVENLABS_VOICE_ID not set")
try:
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{config.ELEVENLABS_VOICE_ID}/stream",
f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json={"text": text, "model_id": config.ELEVENLABS_MODEL_ID},
json={"text": text, "model_id": model_for(text, voice_id)},
timeout=60,
stream=True,
)
@@ -83,6 +113,55 @@ def chunks_to_int16(byte_chunks: Iterable[bytes]) -> Iterator[np.ndarray]:
yield np.frombuffer(data[:usable], dtype=np.int16)
def synthesize_dialogue(
inputs: list, model_id: Optional[str] = None, stability: Optional[float] = None
) -> tuple[np.ndarray, int]:
"""Multi-voice scene via ElevenLabs Text to Dialogue.
One request, one take: the whole exchange is synthesized together, which
is the point — the model hears the previous line, so reactions and timing
land instead of sounding like separately-rendered clips.
Same PCM-over-`requests` posture as the rest of this module (no SDK, no
`play()` shelling out to ffplay), so playback is the same sounddevice path
everything else uses and barge-in works on it unchanged. There is no
documented streaming variant, and a scene is a short set piece anyway, so
this is whole-clip only.
"""
if not (config.ELEVENLABS_API_KEY and inputs):
raise TtsError("ELEVENLABS_API_KEY not set (or no dialogue lines)")
body: dict = {
"inputs": [
{"text": str(entry.get("text") or ""), "voice_id": str(entry.get("voice_id") or "")}
for entry in inputs
],
"model_id": model_id or config.DIALOGUE_MODEL_ID,
}
if stability is not None:
body["settings"] = {"stability": float(stability)}
try:
response = requests.post(
"https://api.elevenlabs.io/v1/text-to-dialogue",
headers={"xi-api-key": config.ELEVENLABS_API_KEY},
params={"output_format": f"pcm_{config.TTS_SAMPLE_RATE}"},
json=body,
timeout=120, # a multi-voice take is slower to render than one line
)
response.raise_for_status()
except Exception as exc:
detail = ""
# The API explains refusals (character limit, unknown voice) in the
# body; surfacing it is what lets Bolt fix the call and retry.
body_text = getattr(getattr(exc, "response", None), "text", "")
if body_text:
detail = f"{body_text[:300]}"
raise TtsError(f"ElevenLabs dialogue request failed: {exc}{detail}") from exc
pcm = np.frombuffer(response.content, dtype=np.int16)
if pcm.size == 0:
raise TtsError("ElevenLabs returned no dialogue audio")
return pcm, config.TTS_SAMPLE_RATE
def play_pcm(pcm: np.ndarray, sample_rate: int, blocking: bool = True, should_stop=None) -> bool:
"""Play a whole clip. Returns True if it finished, False if *should_stop*
(barge-in) cut it short. *should_stop* is polled while audio plays — each
@@ -134,11 +213,12 @@ def speak_offline(text: str) -> None:
engine.runAndWait()
def speak(text: str, on_error=None, should_stop=None) -> bool:
def speak(text: str, on_error=None, should_stop=None, voice_id: Optional[str] = None) -> bool:
"""Speak *text*, preferring streaming ElevenLabs, then whole-clip
ElevenLabs, then offline TTS. *on_error*, if given, is called with the
exception when ElevenLabs fails (useful for logging) — a fallback still
runs either way. Returns False if barge-in interrupted playback.
*voice_id* overrides the configured voice for this line only.
The text is sanitized first (speech_text.for_speech): server replies are
written for a chat window, and a voice reads markdown/emoji literally
@@ -149,12 +229,16 @@ def speak(text: str, on_error=None, should_stop=None) -> bool:
return True
if config.TTS_STREAMING:
try:
return play_stream(stream_pcm(text), config.TTS_SAMPLE_RATE, should_stop=should_stop)
return play_stream(
stream_pcm(text, voice_id=voice_id),
config.TTS_SAMPLE_RATE,
should_stop=should_stop,
)
except TtsError as exc:
if on_error is not None:
on_error(exc)
try:
pcm, sample_rate = synthesize_pcm(text)
pcm, sample_rate = synthesize_pcm(text, voice_id=voice_id)
return play_pcm(pcm, sample_rate, should_stop=should_stop)
except TtsError as exc:
if on_error is not None:
+38 -1
View File
@@ -66,9 +66,38 @@ DEEPGRAM_MODEL = os.environ.get("DEEPGRAM_MODEL", "nova-3")
ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY", "")
ELEVENLABS_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID", "")
ELEVENLABS_MODEL_ID = os.environ.get("ELEVENLABS_MODEL_ID", "eleven_flash_v2")
# eleven_flash_v2 is English-only, and the two cases that swap the voice
# (server-picked `speak_as`, or a reply with non-ASCII in it) are usually
# exactly the cases where the reply isn't English — see tts.model_for().
ELEVENLABS_MULTILINGUAL_MODEL_ID = os.environ.get(
"ELEVENLABS_MULTILINGUAL_MODEL_ID", "eleven_flash_v2_5"
)
# ElevenLabs PCM output formats are named pcm_<sample_rate>.
TTS_SAMPLE_RATE = int(os.environ.get("TTS_SAMPLE_RATE", "24000"))
# Does a voice the server picks (its speak_as marker — "talk like a pirate",
# "say that in Japanese") stay on for later replies, or last one reply only?
# Sticky by default: the server tags a single reply and does *not* keep the
# voice id in its history, so a one-reply-only voice can't be re-used when
# you say "keep talking like that" — it would have to search for a voice
# again. Reset it from the tray ("Use default voice") or by restarting.
VOICE_STICKY = os.environ.get("VOICE_STICKY", "true").lower() in ("1", "true", "yes", "on")
# ── multi-voice dialogue (ElevenLabs Text to Dialogue) ──────────────────────
# Lets Bolt play a short scene in several voices with delivery tags the v3
# model acts on ("[cheerfully] Hello"), instead of one voice reading a line.
# Driven by the server through the `dialoguectl` relayed command — see
# dialogue.py. Costs a separate (slower, whole-clip) request per scene, so
# it's a set piece, not the normal reply path.
#
# DIALOGUE_VOICES names the cast: "narrator:9BWtsMINqrJLrRacOk9x,villain:IKne3meq5aSn9XLyUdCD".
# The name "self" always resolves to the voice the pet is currently using,
# including one the server picked with speak_as.
DIALOGUE = os.environ.get("DIALOGUE", "true").lower() in ("1", "true", "yes", "on")
DIALOGUE_MODEL_ID = os.environ.get("DIALOGUE_MODEL_ID", "eleven_v3")
DIALOGUE_VOICES = os.environ.get("DIALOGUE_VOICES", "")
# ── mic / VAD (same tuning knobs as bolt_desk.py) ───────────────────────────
MIC_DEVICE = os.environ.get("MIC_DEVICE", "") or None # sounddevice name/index
@@ -91,7 +120,7 @@ GRACE_SECONDS = float(os.environ.get("VAD_GRACE_SECONDS", "4"))
# keeps feeding it noise. 0 means no cap.
FOLLOW_UP_LISTEN = os.environ.get("FOLLOW_UP_LISTEN", "true").lower() in ("1", "true", "yes", "on")
FOLLOW_UP_MAX_TURNS = int(os.environ.get("FOLLOW_UP_MAX_TURNS", "3"))
FOLLOW_UP_MAX_TURNS = int(os.environ.get("FOLLOW_UP_MAX_TURNS", "10"))
FOLLOW_UP_GRACE_SECONDS = float(os.environ.get("FOLLOW_UP_GRACE_SECONDS", "7"))
COMMAND_TIMEOUT_SECONDS = int(os.environ.get("COMMAND_TIMEOUT_SECONDS", "30"))
@@ -110,6 +139,14 @@ SUDO_ASKPASS_HELPER = os.environ.get("SUDO_ASKPASS_HELPER", "") # blank = auto-
SUDO_COMMAND_TIMEOUT_SECONDS = int(os.environ.get("SUDO_COMMAND_TIMEOUT_SECONDS", "180"))
HEARTBEAT_INTERVAL_SECONDS = float(os.environ.get("HEARTBEAT_INTERVAL_SECONDS", "60"))
# ── self-restart ────────────────────────────────────────────────────────────
# `petctl self_restart` lets Bolt restart the pet after editing its code, so
# he can see his own change running instead of waiting for someone to restart
# it by hand. The code is import-checked in a subprocess first, and the reason
# is carried across the restart so the new process can report back — see
# self_restart.py. SELF_RESTART_MAX/_WINDOW_SECONDS bound the crash-loop case.
SELF_RESTART = os.environ.get("SELF_RESTART", "true").lower() in ("1", "true", "yes", "on")
# ── barge-in (interrupt playback while the pet is talking) ──────────────────
# The mic stays live while the pet talks. BARGE_IN_MODE decides what counts
# as an interruption:
+211 -6
View File
@@ -20,10 +20,12 @@ from typing import Optional
from PySide6.QtCore import QObject, Signal
from . import (
config, file_delivery, file_ops, history as history_mod,
monitors as monitors_mod, notifications, pet_actions, quiet,
screen_context, screen_text, server_client, speech_text, updater,
config, dialogue as dialogue_mod, file_delivery, file_ops,
history as history_mod, monitors as monitors_mod, notifications,
pet_actions, quiet, screen_context, screen_text, self_restart,
server_client, speech_text, updater,
)
from . import __version__
from .audio import barge_in, mic, stt, tts, wake_word
from .state import PetState, PetStateMachine
@@ -38,6 +40,7 @@ class PetController(QObject):
log = Signal(str)
action = Signal(dict) # parsed petctl action for the UI to perform
napping = Signal(bool) # quiet hours / fullscreen do-not-disturb
voice_changed = Signal(str) # name of the server-picked voice ("" = default)
restart_requested = Signal(str) # version we just updated to
finished = Signal()
@@ -66,6 +69,13 @@ class PetController(QObject):
self._wake_threshold = config.WAKE_WORD_THRESHOLD
self._near_misses = wake_word.NearMissLog()
# The voice the server last picked for us with `speak_as` ("" = the
# configured default). Held here rather than passed straight through
# to one tts.speak() call because it's sticky by default — see
# _apply_voice for why.
self._voice_id = ""
self._voice_name = ""
self._barge_in: Optional[barge_in.BargeInDetector] = None
self._napping = False
self._nap_forced: Optional[bool] = None # petctl nap on/off overrides the schedule
@@ -80,6 +90,9 @@ class PetController(QObject):
self._last_update_check = 0.0
self._update_pending = False # applied on disk, waiting for the restart
# Armed by `petctl self_restart`, fired after the turn it was asked in
# (see _arm_self_restart for why it can't happen inline).
self._restart_context = None
self._notification_watcher: Optional[notifications.NotificationWatcher] = None
self._notification_gate = notifications.NotificationGate(
@@ -117,6 +130,20 @@ class PetController(QObject):
def reset_wake_stats(self) -> None:
self._near_misses.clear()
def current_voice(self) -> str:
"""Name (or id) of the server-picked voice in use, "" for the default."""
return self._voice_name or self._voice_id
def reset_voice(self) -> None:
"""Drop a server-picked voice and go back to Bolt's own. The tray's
way out of a voice you didn't want to keep — the server has no way to
ask for the default back, since it never learns what it is."""
if not self._voice_id:
return
self._voice_id = self._voice_name = ""
self.log.emit("Voice: back to the default.")
self.voice_changed.emit("")
def stop(self) -> None:
self._running = False
self._talk_now.set() # wake up anything blocked waiting on it
@@ -167,6 +194,7 @@ class PetController(QObject):
except Exception as exc:
self.log.emit(f"Server not reachable yet ({exc}) — will keep trying per-request.")
self._start_notification_bridge()
self._report_self_restart()
self._loop()
if self._notification_watcher is not None:
self._notification_watcher.stop()
@@ -260,8 +288,10 @@ class PetController(QObject):
return
self._check_deliveries()
self._speak(reply)
self._apply_voice(reply)
self._speak(reply.text)
self._state.transition(PetState.IDLE)
self._maybe_self_restart()
def _with_context(self, text: str) -> str:
"""Everything the server gets alongside what you actually said: the
@@ -307,6 +337,18 @@ class PetController(QObject):
kind = action["action"]
if kind == "monitors":
return monitors_mod.describe(self._monitors, self._pet_monitor)
if kind == "self_restart":
return self._arm_self_restart(action.get("reason") or "")
if kind == "voice":
# Answered here, not by describe(): the UI has no part in it,
# and the server needs to hear whether there was anything to
# drop — it can't see which voice we're using.
previous = self.current_voice()
self.reset_voice()
return (
f"[pet] back to your own voice (was {previous})" if previous
else "[pet] already using your own voice"
)
if kind == "read":
return self._read_screen(action["target"])
if kind == "jump":
@@ -327,6 +369,14 @@ class PetController(QObject):
self.action.emit(action)
return pet_actions.describe(action)
try:
scene = dialogue_mod.parse(command)
except dialogue_mod.DialogueError as exc:
self.log.emit(f"dialoguectl: {exc}")
return f"[dialogue] {exc}"
if scene is not None:
return self._play_dialogue(scene)
try:
file_action = file_ops.parse(command)
except file_ops.FileOpError as exc:
@@ -365,6 +415,159 @@ class PetController(QObject):
self.log.emit(f"Reading monitor {monitor.number} ({monitor.name})…")
return screen_text.read_monitor(monitor, limit)
def _arm_self_restart(self, reason: str) -> str:
"""`petctl self_restart` — check the code, then arm a restart.
Nothing restarts here. The tool result has to get back up the relay
before this process can die (otherwise the server waits out its
timeout on a turn that will never finish), so the restart is armed and
`_maybe_self_restart` fires it once the turn has been spoken. The
preflight import runs *now*, in this turn, so a syntax error Bolt just
introduced comes back as something he can read and fix rather than as
a pet that never comes back."""
if not config.SELF_RESTART:
return "[pet] self-restart is disabled on this device (SELF_RESTART=false)"
if self._restart_context is not None:
return "[pet] a restart is already armed for the end of this turn"
try:
self_restart.check_loop_guard(self_restart.load())
self.log.emit("Self-restart requested — checking the code imports first…")
self_restart.preflight()
except self_restart.RestartError as exc:
self.log.emit(f"Self-restart refused: {exc}")
return f"[pet] restart refused — {exc}"
recent = [entry.text[:120] for entry in self.history.entries()[-4:]]
self._restart_context = self_restart.arm(
reason or "no reason given",
verify=reason,
version=__version__,
session=config.SESSION_ID,
recent=recent,
)
self.log.emit("Self-restart armed; it happens after this turn.")
return (
"[pet] code imports cleanly; restarting as soon as this turn finishes. "
"I'll come back and tell you what version I'm on and what I found — "
"wrap up your reply now, the next thing you hear from me is the report."
)
def _maybe_self_restart(self) -> bool:
"""Fire an armed restart, once the turn is over and the reply spoken.
Returns True if a restart was requested, so the caller can stop
driving the pipeline — the process is on its way out."""
if self._restart_context is None:
return False
self._update_pending = True # same latch the updater uses: no double restart
self.log.emit("Restarting now.")
self._state.force(PetState.IDLE)
self.restart_requested.emit(f"self-restart: {self._restart_context.reason[:60]}")
return True
def _report_self_restart(self) -> None:
"""On the way up: tell the server we're back, and why we left.
Runs once, before the listen loop starts, and only when a context file
was left behind. The report goes through the ordinary conversation
path, so Bolt's answer is spoken out loud like any other turn — which
is what makes "restart and check the sprites load" finish as a
sentence instead of a silence."""
context = self_restart.load()
if context is None:
return
self_restart.clear()
message = self_restart.report(context, version=__version__)
self.log.emit(f"Back from a self-restart ({context.reason[:80]}).")
self.history.add(history_mod.SYSTEM, message, time.time())
try:
reply = server_client.converse(message, on_command=self._handle_command)
except server_client.ServerError as exc:
# The restart still worked; only the report failed. Say so locally
# rather than pretending nothing happened.
self.log.emit(f"Couldn't report the restart to the server: {exc}")
return
self._apply_voice(reply)
if reply.text.strip() and not self._napping:
self._speak(reply.text)
self._state.force(PetState.IDLE)
def _play_dialogue(self, scene: dict) -> str:
"""Play a `dialoguectl` scene and report back up the relay.
This runs *mid-turn* (the server is still waiting on the tool result),
so the pet has to look like it's talking and then go back to waiting —
hence the TALKING → THINKING leg rather than the usual return to IDLE.
Everything a normal reply gets, a scene gets too: the bubble, the
transcript, and barge-in, so a long scene can be talked over exactly
like a long answer."""
if not config.DIALOGUE:
return "[dialogue] disabled on this device (DIALOGUE=false)"
try:
inputs = dialogue_mod.resolve(
scene,
voices=dialogue_mod.parse_voice_map(config.DIALOGUE_VOICES),
self_voice=self._voice_id or config.ELEVENLABS_VOICE_ID,
)
except dialogue_mod.DialogueError as exc:
self.log.emit(f"dialoguectl: {exc}")
return f"[dialogue] {exc}"
text = dialogue_mod.spoken_text(scene)
self.log.emit(f"Dialogue ({len(inputs)} lines): {text[:120]}")
try:
pcm, sample_rate = tts.synthesize_dialogue(
inputs, model_id=scene.get("model"), stability=scene.get("stability")
)
except tts.TtsError as exc:
self.log.emit(f"Dialogue failed: {exc}")
# Reported, not raised: the server can read this, shorten the
# scene or fix the voice, and try again inside the same turn.
return f"[dialogue] couldn't synthesize it: {exc}"
resume = self._state.state
self._state.transition(PetState.TALKING)
self.said.emit(speech_text.for_display(text))
self.history.add(history_mod.PET, text, time.time())
should_stop = None
if self._barge_in is not None:
self._barge_in.reset()
should_stop = self._barge_in.check
completed = tts.play_pcm(pcm, sample_rate, should_stop=should_stop)
if self._barge_in is not None:
self._barge_in.reset() # the pet's own voices are in the wake window
if resume in (PetState.THINKING, PetState.IDLE):
self._state.transition(resume)
if not completed:
return dialogue_mod.describe(scene) + " (interrupted — they talked over it)"
return dialogue_mod.describe(scene)
def _apply_voice(self, reply) -> None:
"""Adopt (or drop) the voice the server tagged this reply with.
The server's `speak_as` marker names an ElevenLabs voice it just
picked — and it adds a Voice Library pick to the account first, so by
the time the id gets here it's usable for TTS. It tags *one* reply,
but the voice sticks by default: the server strips the marker before
storing the turn, so it can't recall the id later, and "keep talking
like that" would otherwise send it searching for a voice all over
again. `VOICE_STICKY=false` makes each pick last exactly one reply.
Untagged replies never *change* the voice — with stickiness on they
just keep whatever's in use, which is what makes the rest of the
conversation stay in the requested voice."""
voice_id = getattr(reply, "voice_id", "")
if voice_id:
if voice_id != self._voice_id:
self._voice_id = voice_id
self._voice_name = getattr(reply, "voice_name", "") or ""
self.log.emit(f"Voice: {self.current_voice()}")
self.voice_changed.emit(self.current_voice())
elif not config.VOICE_STICKY:
self.reset_voice()
def _speak(self, text: str) -> None:
self._state.transition(PetState.TALKING)
# Bubble gets the markdown stripped but emoji kept (it can't render
@@ -382,6 +585,7 @@ class PetController(QObject):
text,
on_error=lambda exc: self.log.emit(f"TTS failed: {exc}"),
should_stop=should_stop,
voice_id=self._voice_id or None,
)
# Read the scoring history *before* resetting, or the log reports the
# blank counters instead of what actually fired.
@@ -504,8 +708,9 @@ class PetController(QObject):
self.log.emit(f"Couldn't forward notification: {exc}")
return
self._check_deliveries()
if reply.strip():
self._speak(reply)
self._apply_voice(reply)
if reply.text.strip():
self._speak(reply.text)
self._state.transition(PetState.IDLE)
# ── file delivery ────────────────────────────────────────────────────
+219
View File
@@ -0,0 +1,219 @@
"""`dialoguectl` — multi-voice dialogue playback (ElevenLabs Text to Dialogue).
Normal replies are one voice saying one thing (audio/tts.py). This is the
other mode: a short *scene* — two or more voices, with delivery tags the v3
model acts on (`[cheerfully]`, `[stuttering]`, `[whispering]`) — synthesized
as a single take so the timing and reactions between lines actually sound
like a conversation rather than clips glued together.
Wire format, the same discipline as file_ops.py and for the same reason: it
rides the server's ordinary `command` tool marker, whose extractor only
captures up to the next newline, so the payload is a **single-line compact
JSON object**.
dialoguectl {"lines": [{"voice": "self", "text": "[cheerfully] Morning!"},
{"voice": "narrator", "text": "[whispering] He lies."}]}
The ElevenLabs field names are accepted too (`inputs` / `voice_id`), because
the model has read that API and copying its shape is the obvious thing to
try:
dialoguectl {"inputs": [{"voice_id": "9BWtsMINqrJLrRacOk9x", "text": "hi"}]}
Voices are *named*, not pasted as ids. `DIALOGUE_VOICES` in .env maps names
to ids (`narrator:9BWts…,villain:IKne3…`), and `self` always means the voice
the pet is speaking with right now — including a voice the server picked
mid-conversation with `speak_as`, so a scene featuring Bolt sounds like
whoever Bolt currently is.
Pure parsing and validation here; the HTTP call is
`audio/tts.synthesize_dialogue` and the playback/state handling is
`controller._play_dialogue`, matching the parse/execute split used by
pet_actions.py and file_ops.py.
The API's own limits are enforced *here*, before the request goes out, so a
mistake comes back through the tool-result relay as a sentence Bolt can act
on ("too many characters, split it") rather than as an HTTP 422 he can't see.
"""
from __future__ import annotations
import json
import re
from typing import Iterable, Optional
_PREFIXES = ("dialoguectl", "dialogue", "scene")
# ElevenLabs Text to Dialogue limits (docs, 2026-07): at most 10 distinct
# voice ids per request and ~2000 characters across all inputs.
MAX_VOICES = 10
MAX_CHARS = 2000
# Names that always mean "the voice the pet is using right now".
SELF_NAMES = ("self", "bolt", "me", "pet")
# A raw ElevenLabs voice id: 20 URL-safe characters, no separators. Used to
# tell "the model pasted an id" from "the model used a name".
_VOICE_ID_RE = re.compile(r"^[A-Za-z0-9]{20}$")
class DialogueError(Exception):
"""Bad dialoguectl syntax or an unusable request — reported back to the
server as this command's output."""
def is_dialogue_command(command: str) -> bool:
parts = (command or "").strip().split(None, 1)
return bool(parts) and parts[0].lower() in _PREFIXES
def parse(command: str) -> Optional[dict]:
"""Parse `dialoguectl <json>` into {"lines": [{"voice", "text"}], ...}.
Returns None if this isn't a dialogue command at all (the caller then
tries filectl, then a real shell command). Raises DialogueError on a
dialogue command that doesn't make sense."""
if not is_dialogue_command(command):
return None
_, _, payload = (command or "").strip().partition(" ")
payload = payload.strip()
if not payload:
raise DialogueError(
'dialoguectl needs a JSON argument, e.g. dialoguectl {"lines": '
'[{"voice": "self", "text": "[cheerfully] hello"}]}'
)
try:
data = json.loads(payload)
except json.JSONDecodeError as exc:
raise DialogueError(
f"couldn't parse the JSON ({exc}). It must be one line of compact "
"JSON — put line breaks inside text as \\n, never as real newlines."
) from exc
if not isinstance(data, dict):
raise DialogueError("the argument must be a JSON object, not a list or a bare value")
raw_lines = data.get("lines")
if raw_lines is None:
raw_lines = data.get("inputs") # the ElevenLabs field name
if not isinstance(raw_lines, list) or not raw_lines:
raise DialogueError('needs a non-empty "lines" array of {"voice", "text"} objects')
lines: list[dict] = []
for index, entry in enumerate(raw_lines, start=1):
if not isinstance(entry, dict):
raise DialogueError(f"line {index} must be an object with 'voice' and 'text'")
text = str(entry.get("text") or "").strip()
if not text:
raise DialogueError(f"line {index} has no text")
voice = str(entry.get("voice") or entry.get("voice_id") or "self").strip()
lines.append({"voice": voice, "text": text})
action = {"action": "dialogue", "lines": lines}
model = str(data.get("model") or data.get("model_id") or "").strip()
if model:
action["model"] = model
stability = data.get("stability")
if stability is not None:
try:
action["stability"] = min(1.0, max(0.0, float(stability)))
except (TypeError, ValueError):
raise DialogueError("stability must be a number between 0 and 1") from None
return action
def parse_voice_map(spec: str) -> dict[str, str]:
"""Parse DIALOGUE_VOICES ("narrator:9BWts…, villain:IKne3…") into a map.
Malformed entries are skipped rather than raising: a typo in .env should
cost that one voice, not the whole feature."""
voices: dict[str, str] = {}
for chunk in str(spec or "").split(","):
name, separator, voice_id = chunk.partition(":")
name, voice_id = name.strip().lower(), voice_id.strip()
if separator and name and voice_id:
voices[name] = voice_id
return voices
def resolve(
action: dict,
*,
voices: Optional[dict] = None,
self_voice: str = "",
) -> list[dict]:
"""Turn parsed lines into the API's `inputs`, resolving names to ids.
*self_voice* is the pet's current voice (which may be a `speak_as` pick,
not the configured default), so "self" tracks whoever Bolt sounds like
right now."""
known = dict(voices or {})
resolved: list[dict] = []
for index, line in enumerate(action.get("lines") or [], start=1):
name = str(line.get("voice") or "self")
key = name.lower()
if key in SELF_NAMES:
voice_id = self_voice
if not voice_id:
raise DialogueError(
"no voice is configured for the pet itself — set "
"ELEVENLABS_VOICE_ID, or name a voice from DIALOGUE_VOICES"
)
elif key in known:
voice_id = known[key]
elif _VOICE_ID_RE.match(name):
voice_id = name # a raw id pasted straight from the voice library
else:
available = ", ".join(sorted(known) + list(SELF_NAMES[:1])) or "self"
raise DialogueError(
f"line {index}: unknown voice {name!r}. Known names: {available}. "
"Use one of those, 'self' for your own voice, or a raw voice id."
)
resolved.append({"text": str(line.get("text") or ""), "voice_id": voice_id})
check_limits(resolved)
return resolved
def check_limits(inputs: Iterable[dict], *, max_voices: int = MAX_VOICES,
max_chars: int = MAX_CHARS) -> None:
"""Enforce the API's own limits before spending a request on a 422."""
entries = list(inputs)
if not entries:
raise DialogueError("no lines to speak")
distinct = {entry["voice_id"] for entry in entries}
if len(distinct) > max_voices:
raise DialogueError(
f"{len(distinct)} different voices — the limit is {max_voices} per scene"
)
total = sum(len(entry["text"]) for entry in entries)
if total > max_chars:
raise DialogueError(
f"{total} characters — the limit is {max_chars} per scene. "
"Split it into two dialoguectl calls."
)
def spoken_text(action: dict) -> str:
"""The scene as readable text, for the speech bubble and the transcript.
Delivery tags are stripped: `[cheerfully]` is a stage direction for the
model, not something to show (or, via tts.speak's sanitizer, to read out)."""
parts = []
for line in action.get("lines") or []:
text = re.sub(r"\[[^\]]{1,40}\]", " ", str(line.get("text") or ""))
text = " ".join(text.split())
if text:
parts.append(text)
return " ".join(parts)
def describe(action: dict, *, played: bool = True) -> str:
"""The tool-result string handed back to the server."""
lines = action.get("lines") or []
voices = sorted({str(line.get("voice") or "self") for line in lines})
if not played:
return f"[dialogue] not played ({len(lines)} lines)"
return (
f"[dialogue] played {len(lines)} line{'s' if len(lines) != 1 else ''} "
f"in {len(voices)} voice{'s' if len(voices) != 1 else ''}: {', '.join(voices)}"
)
+25 -1
View File
@@ -44,9 +44,17 @@ HELP = (
"petctl emote <" + "|".join(EMOTES) + ">\n"
"petctl say <text>\n"
"petctl wander on|off\n"
"petctl nap on|off"
"petctl nap on|off\n"
"petctl voice reset\n"
"petctl self_restart [why]"
)
# `petctl voice` only ever goes one way: back to the configured voice. Picking
# a *different* one is the server's job (its speak_as reply marker), and it
# already knows how — what it has no way to say is "never mind, be yourself
# again", because it was never told which voice that is.
VOICE_RESETS = ("reset", "default", "normal", "own", "back", "mine", "yours")
class ActionError(Exception):
"""Bad petctl syntax — reported back to the server as command output."""
@@ -129,6 +137,22 @@ def parse(command: str) -> Optional[dict]:
raise ActionError("wander needs on or off")
return {"action": "wander", "enabled": _bool_arg(args[0])}
if verb == "voice":
target = (args[0].lower() if args else "reset").lstrip("-")
if target not in VOICE_RESETS:
raise ActionError(
f"can't set a voice from petctl (got {args[0]!r}); "
"use the speak_as reply marker to pick one. "
"petctl voice reset goes back to the default voice."
)
return {"action": "voice", "voice": "default"}
if verb in ("self_restart", "restart", "reboot"):
# Free text, not a fixed grammar: the argument is a note to the pet's
# *next* process about why it died and what to look at when it comes
# back, so anything the model wants to tell future-itself is valid.
return {"action": "self_restart", "reason": " ".join(args).strip()}
if verb in ("nap", "sleep", "dnd"):
if not args:
raise ActionError("nap needs on or off")
+235
View File
@@ -0,0 +1,235 @@
"""`petctl self_restart` — the pet restarting itself, and remembering why.
Bolt can already edit this repo through `filectl` and run commands through the
shell relay, which means he can change the pet's own code. What he could not
do is *see the result*: the running process keeps the old modules in memory,
so an edit is invisible until somebody restarts the pet by hand, and by then
the conversation that motivated it is over. That makes the edit-test-review
loop a human errand.
This closes the loop. The tricky part is that the thing being asked to report
back is the thing that dies, so the mechanism is built around three problems:
1. **The turn must survive.** A restart mid-turn would kill the HTTP tool
relay before the result was posted, and the server would sit waiting until
it timed out — the conversation lost, with no explanation. So the command
only *arms* the restart: it returns immediately, the turn finishes and Bolt
speaks his reply, and the restart happens after (see
`controller._maybe_self_restart`), exactly like the updater's "only between
turns" rule.
2. **A broken edit must not be fatal.** Before anything is armed, the new code
is imported in a *subprocess* (`preflight`) — this process still holds the
old modules, so importing here would prove nothing. A syntax error comes
back as the command's output, in the same turn, and nothing restarts. That
is the difference between "Bolt broke the pet and lost his own way to fix
it" and "Bolt got a traceback and tried again".
3. **The reason must outlive the process.** The context (why, what to check,
which version, when) is written to disk before exec and read on the way
back up, so the new process can open with "I'm back — you asked me to check
X" instead of amnesia. That report goes to the server as a normal turn, so
Bolt sees the result of his own change and can carry on.
A loop guard bounds the worst case: `MAX_RESTARTS` inside `WINDOW_SECONDS`
and further self-restarts are refused with a reason, so an edit-restart-crash
cycle stops on its own rather than spinning the process forever.
Pure-ish and injectable throughout (paths, clock, subprocess runner) so the
whole thing is testable without ever restarting anything.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import time
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any, Callable, Optional
from . import config
# Lives in the cache dir, not the repo: it is transient state about *this*
# machine's process, and it must never end up in a git diff of the checkout
# Bolt is editing.
DEFAULT_STATE_PATH = Path.home() / ".cache" / "bolt-pet" / "restart_context.json"
# Loop guard. Deliberately small: a healthy edit-check cycle is one restart
# per change, and anything hammering past this is a crash loop, not work.
MAX_RESTARTS = int(os.environ.get("SELF_RESTART_MAX", "5"))
WINDOW_SECONDS = float(os.environ.get("SELF_RESTART_WINDOW_SECONDS", "900"))
# What the preflight subprocess imports. `ui.app` pulls in the widest slice of
# the package (Qt, controller, audio, every helper), so if this imports, a
# restart will at least reach the event loop.
_PREFLIGHT_IMPORT = "import bolt_pet, bolt_pet.controller, bolt_pet.ui.app"
class RestartError(Exception):
"""A refused restart — reported back to the server as command output."""
@dataclass
class RestartContext:
"""What the dying process wants the next one to know."""
reason: str = ""
verify: str = ""
armed_at: float = 0.0
version: str = ""
session: str = ""
recent: list = field(default_factory=list)
restarts: list = field(default_factory=list) # timestamps, for the loop guard
def as_dict(self) -> dict[str, Any]:
return asdict(self)
def _now() -> float:
return time.time()
def load(path: Optional[Path] = None) -> Optional[RestartContext]:
"""Read the context left by a previous process, or None."""
target = Path(path or DEFAULT_STATE_PATH)
try:
data = json.loads(target.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError, OSError):
return None
if not isinstance(data, dict):
return None
known = {field_name for field_name in RestartContext().as_dict()}
return RestartContext(**{k: v for k, v in data.items() if k in known})
def save(context: RestartContext, path: Optional[Path] = None) -> None:
"""Persist the context atomically — a half-written file on the way out
would make the next process start confused instead of oriented."""
target = Path(path or DEFAULT_STATE_PATH)
target.parent.mkdir(parents=True, exist_ok=True)
descriptor, temp_path = tempfile.mkstemp(dir=target.parent, prefix=".restart_", suffix=".tmp")
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
json.dump(context.as_dict(), handle, ensure_ascii=False, indent=1)
os.replace(temp_path, target)
except BaseException:
try:
os.unlink(temp_path)
except OSError:
pass
raise
def clear(path: Optional[Path] = None) -> None:
"""Consume the context. Called once it has been reported, so the pet
doesn't announce the same restart every time it starts."""
try:
Path(path or DEFAULT_STATE_PATH).unlink()
except (FileNotFoundError, OSError):
pass
def recent_restarts(context: Optional[RestartContext], *, now: Optional[float] = None) -> list:
current = now if now is not None else _now()
stamps = list((context.restarts if context else []) or [])
return [stamp for stamp in stamps if current - float(stamp) <= WINDOW_SECONDS]
def check_loop_guard(context: Optional[RestartContext], *, now: Optional[float] = None) -> None:
"""Refuse to restart if we've already done it too many times recently."""
stamps = recent_restarts(context, now=now)
if len(stamps) >= MAX_RESTARTS:
raise RestartError(
f"refusing: {len(stamps)} self-restarts in the last "
f"{int(WINDOW_SECONDS / 60)} minutes. Something is looping — fix the "
"cause, or wait for the window to clear before trying again."
)
def preflight(
repo: Optional[Path] = None,
run: Optional[Callable[..., Any]] = None,
timeout: float = 120.0,
) -> None:
"""Import the current source in a subprocess; raise if it's broken.
This process has the *old* modules loaded, so importing in-process would
happily succeed on a file that no longer parses. Mirrors
`updater._smoke_test`, and exists for the same reason: never hand the
session to code that can't start."""
runner = run or subprocess.run
root = Path(repo or config.HERE)
try:
completed = runner(
[sys.executable, "-c", _PREFLIGHT_IMPORT],
cwd=str(root), capture_output=True, text=True, timeout=timeout,
env={**os.environ, "QT_QPA_PLATFORM": "offscreen"}, # no display needed to import
)
except Exception as exc: # subprocess itself failed to run
raise RestartError(f"couldn't run the preflight import check: {exc}") from exc
if completed.returncode != 0:
detail = (completed.stderr or completed.stdout or "").strip()
raise RestartError(
"the current code does not import, so restarting would leave you with "
f"nothing running. Fix this first:\n{detail[-800:]}"
)
def arm(
reason: str,
*,
verify: str = "",
version: str = "",
session: str = "",
recent: Optional[list] = None,
path: Optional[Path] = None,
now: Optional[float] = None,
) -> RestartContext:
"""Record why we're about to die, carrying the restart history forward."""
current = now if now is not None else _now()
previous = load(path)
context = RestartContext(
reason=" ".join(str(reason or "").split())[:400],
verify=" ".join(str(verify or "").split())[:400],
armed_at=current,
version=str(version or ""),
session=str(session or ""),
recent=list(recent or [])[-6:],
restarts=recent_restarts(previous, now=current) + [current],
)
save(context, path)
return context
def report(
context: RestartContext,
*,
version: str = "",
now: Optional[float] = None,
) -> str:
"""The message the new process sends the server on the way up.
Phrased as Bolt reporting to himself, because that is what it is: the
server sees it as an ordinary turn, and the reply comes back through the
normal pipeline — which is what lets "restart and check X" finish as a
sentence spoken out loud."""
current = now if now is not None else _now()
took = max(0.0, current - float(context.armed_at or current))
lines = [
"[pet self-restart] I restarted myself and I'm back up.",
f"- reason: {context.reason or 'not recorded'}",
f"- took: {took:.1f}s",
f"- version now running: {version or 'unknown'}"
+ (f" (was {context.version})" if context.version and context.version != version else ""),
]
if context.verify:
lines.append(f"- you wanted to check: {context.verify}")
if context.recent:
lines.append("- what we were doing before: " + " | ".join(str(x)[:120] for x in context.recent))
lines.append(
"The new code is loaded and running. If you wanted to verify something, "
"check it now (filectl to read, command to test) and tell the user what you found."
)
return "\n".join(lines)
+23 -3
View File
@@ -9,6 +9,11 @@ memory, tools, and persona as Discord chat and the Linux voice client:
... -> POST /desk/tool_result (repeat until the server sends a reply)
reply <- returned to caller
A reply can also carry a voice (`voice_id`/`voice_name`), which is how the
server's `speak_as` marker reaches us: Bolt searched the ElevenLabs voice
library, picked one, and tagged the reply with it — the client is what
actually speaks in it. See `Reply` and controller._apply_voice.
Kept dependency-free beyond `requests` so it's easy to unit test with mocks.
"""
@@ -16,7 +21,7 @@ from __future__ import annotations
import subprocess
from pathlib import Path
from typing import Callable, Optional
from typing import Callable, NamedTuple, Optional
import requests
@@ -29,6 +34,17 @@ class ServerError(Exception):
"""Raised when the server responds with an error payload or unreachable."""
class Reply(NamedTuple):
"""One final reply from the desk API. *voice_id* is set only when the
server tagged this reply with a `speak_as` voice; *voice_name* is the
human-readable name that came with it (may be empty even when the id
isn't). Both empty means "say it in the usual voice"."""
text: str
voice_id: str = ""
voice_name: str = ""
def _headers() -> dict:
return {"X-Desk-Api-Key": config.API_KEY}
@@ -74,7 +90,7 @@ def converse(
text: str,
on_command: Callable[[str], str] = run_local_command,
timeout: float = 120.0,
) -> str:
) -> Reply:
"""Send one turn of conversation to the desk API, relaying any commands
the server sends back until it produces a final reply.
@@ -111,7 +127,11 @@ def converse(
raise ServerError(f"couldn't reach the server during tool relay: {exc}") from exc
if payload.get("type") == "reply":
return str(payload.get("text") or "")
return Reply(
text=str(payload.get("text") or ""),
voice_id=str(payload.get("voice_id") or ""),
voice_name=str(payload.get("voice_name") or ""),
)
raise ServerError(str(payload.get("error") or "unknown server response"))
+6 -1
View File
@@ -26,11 +26,16 @@ class PetState(str, Enum):
# make the pet speak unprompted — a reminder firing, a nudge from the server
# — without the user having said anything first, so there's no preceding
# LISTENING/THINKING leg for that turn.
#
# TALKING -> THINKING is the mirror case: a `dialoguectl` scene is played
# *mid-turn*, while the server is still waiting on the tool result, so the pet
# talks and then goes back to waiting rather than falling to IDLE (which would
# make it look like the turn had ended).
_TRANSITIONS: dict[PetState, set[PetState]] = {
PetState.IDLE: {PetState.LISTENING, PetState.TALKING, PetState.ERROR},
PetState.LISTENING: {PetState.THINKING, PetState.IDLE, PetState.ERROR},
PetState.THINKING: {PetState.TALKING, PetState.IDLE, PetState.ERROR},
PetState.TALKING: {PetState.IDLE, PetState.ERROR},
PetState.TALKING: {PetState.IDLE, PetState.THINKING, PetState.ERROR},
PetState.ERROR: {PetState.IDLE},
}
+4
View File
@@ -80,6 +80,7 @@ def run() -> int:
on_set_nap=_set_nap,
on_show_history=history_window.show_refreshed,
on_show_wake_tuner=tuner_window.show_refreshed,
on_reset_voice=controller.reset_voice,
)
def _handle_napping(napping: bool) -> None:
@@ -87,6 +88,9 @@ def run() -> int:
tray.set_napping(napping)
controller.napping.connect(_handle_napping)
# The server can hand Bolt a different voice mid-conversation (speak_as);
# the tray is where you get his own back.
controller.voice_changed.connect(tray.set_voice)
# Push-to-talk: a global hook, because the pet window never has focus.
# request_talk_now() only sets a threading.Event, so it's safe to call
+23 -1
View File
@@ -1,6 +1,7 @@
"""System tray icon — the pet window is frameless with no taskbar entry, so
this menu is the only always-available way to control or exit it: talk now,
mute, wander, click-through, nap, history, wake-word tuning, quit.
mute, wander, click-through, nap, history, wake-word tuning, voice reset,
quit.
Every entry is a plain callback passed in by ui/app.py; this file knows
nothing about the controller or the pet window.
@@ -46,6 +47,7 @@ class PetTray(QSystemTrayIcon):
on_set_nap: Optional[Callable[[bool], None]] = None,
on_show_history: Optional[Callable[[], None]] = None,
on_show_wake_tuner: Optional[Callable[[], None]] = None,
on_reset_voice: Optional[Callable[[], None]] = None,
parent=None,
):
super().__init__(_make_icon(muted=False), parent)
@@ -102,6 +104,16 @@ class PetTray(QSystemTrayIcon):
tuner_action.triggered.connect(on_show_wake_tuner)
menu.addAction(tuner_action)
# Only ever enabled while a server-picked voice (speak_as) is in use —
# it's the way back from "talk like a pirate", which nothing else
# undoes short of a restart.
self._voice_action = None
if on_reset_voice is not None:
self._voice_action = QAction("Use default voice", menu)
self._voice_action.setEnabled(False)
self._voice_action.triggered.connect(on_reset_voice)
menu.addAction(self._voice_action)
menu.addSeparator()
quit_action = QAction("Quit", menu)
quit_action.triggered.connect(on_quit)
@@ -115,6 +127,16 @@ class PetTray(QSystemTrayIcon):
self._mute_action.setChecked(self._muted)
self._refresh_icon()
def set_voice(self, voice: str) -> None:
"""Reflect the voice the controller is speaking in — a name (or id)
when the server picked one, "" for Bolt's own."""
if self._voice_action is None:
return
self._voice_action.setEnabled(bool(voice))
self._voice_action.setText(
f"Use default voice (now: {voice})" if voice else "Use default voice"
)
def set_napping(self, napping: bool) -> None:
"""Reflect a nap the *controller* decided on (quiet hours, fullscreen,
or a petctl command) — not just ones clicked here."""