236 lines
9.4 KiB
Python
236 lines
9.4 KiB
Python
"""`petctl self_restart` — the pet restarting itself, and remembering why.
|
|
|
|
Bolt can already edit this repo through `filectl` and run commands through the
|
|
shell relay, which means he can change the pet's own code. What he could not
|
|
do is *see the result*: the running process keeps the old modules in memory,
|
|
so an edit is invisible until somebody restarts the pet by hand, and by then
|
|
the conversation that motivated it is over. That makes the edit-test-review
|
|
loop a human errand.
|
|
|
|
This closes the loop. The tricky part is that the thing being asked to report
|
|
back is the thing that dies, so the mechanism is built around three problems:
|
|
|
|
1. **The turn must survive.** A restart mid-turn would kill the HTTP tool
|
|
relay before the result was posted, and the server would sit waiting until
|
|
it timed out — the conversation lost, with no explanation. So the command
|
|
only *arms* the restart: it returns immediately, the turn finishes and Bolt
|
|
speaks his reply, and the restart happens after (see
|
|
`controller._maybe_self_restart`), exactly like the updater's "only between
|
|
turns" rule.
|
|
2. **A broken edit must not be fatal.** Before anything is armed, the new code
|
|
is imported in a *subprocess* (`preflight`) — this process still holds the
|
|
old modules, so importing here would prove nothing. A syntax error comes
|
|
back as the command's output, in the same turn, and nothing restarts. That
|
|
is the difference between "Bolt broke the pet and lost his own way to fix
|
|
it" and "Bolt got a traceback and tried again".
|
|
3. **The reason must outlive the process.** The context (why, what to check,
|
|
which version, when) is written to disk before exec and read on the way
|
|
back up, so the new process can open with "I'm back — you asked me to check
|
|
X" instead of amnesia. That report goes to the server as a normal turn, so
|
|
Bolt sees the result of his own change and can carry on.
|
|
|
|
A loop guard bounds the worst case: `MAX_RESTARTS` inside `WINDOW_SECONDS`
|
|
and further self-restarts are refused with a reason, so an edit-restart-crash
|
|
cycle stops on its own rather than spinning the process forever.
|
|
|
|
Pure-ish and injectable throughout (paths, clock, subprocess runner) so the
|
|
whole thing is testable without ever restarting anything.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional
|
|
|
|
from . import config
|
|
|
|
# Lives in the cache dir, not the repo: it is transient state about *this*
|
|
# machine's process, and it must never end up in a git diff of the checkout
|
|
# Bolt is editing.
|
|
DEFAULT_STATE_PATH = Path.home() / ".cache" / "bolt-pet" / "restart_context.json"
|
|
|
|
# Loop guard. Deliberately small: a healthy edit-check cycle is one restart
|
|
# per change, and anything hammering past this is a crash loop, not work.
|
|
MAX_RESTARTS = int(os.environ.get("SELF_RESTART_MAX", "5"))
|
|
WINDOW_SECONDS = float(os.environ.get("SELF_RESTART_WINDOW_SECONDS", "900"))
|
|
|
|
# What the preflight subprocess imports. `ui.app` pulls in the widest slice of
|
|
# the package (Qt, controller, audio, every helper), so if this imports, a
|
|
# restart will at least reach the event loop.
|
|
_PREFLIGHT_IMPORT = "import bolt_pet, bolt_pet.controller, bolt_pet.ui.app"
|
|
|
|
|
|
class RestartError(Exception):
|
|
"""A refused restart — reported back to the server as command output."""
|
|
|
|
|
|
@dataclass
|
|
class RestartContext:
|
|
"""What the dying process wants the next one to know."""
|
|
|
|
reason: str = ""
|
|
verify: str = ""
|
|
armed_at: float = 0.0
|
|
version: str = ""
|
|
session: str = ""
|
|
recent: list = field(default_factory=list)
|
|
restarts: list = field(default_factory=list) # timestamps, for the loop guard
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
|
|
def _now() -> float:
|
|
return time.time()
|
|
|
|
|
|
def load(path: Optional[Path] = None) -> Optional[RestartContext]:
|
|
"""Read the context left by a previous process, or None."""
|
|
target = Path(path or DEFAULT_STATE_PATH)
|
|
try:
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
|
return None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
known = {field_name for field_name in RestartContext().as_dict()}
|
|
return RestartContext(**{k: v for k, v in data.items() if k in known})
|
|
|
|
|
|
def save(context: RestartContext, path: Optional[Path] = None) -> None:
|
|
"""Persist the context atomically — a half-written file on the way out
|
|
would make the next process start confused instead of oriented."""
|
|
target = Path(path or DEFAULT_STATE_PATH)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor, temp_path = tempfile.mkstemp(dir=target.parent, prefix=".restart_", suffix=".tmp")
|
|
try:
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
json.dump(context.as_dict(), handle, ensure_ascii=False, indent=1)
|
|
os.replace(temp_path, target)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(temp_path)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def clear(path: Optional[Path] = None) -> None:
|
|
"""Consume the context. Called once it has been reported, so the pet
|
|
doesn't announce the same restart every time it starts."""
|
|
try:
|
|
Path(path or DEFAULT_STATE_PATH).unlink()
|
|
except (FileNotFoundError, OSError):
|
|
pass
|
|
|
|
|
|
def recent_restarts(context: Optional[RestartContext], *, now: Optional[float] = None) -> list:
|
|
current = now if now is not None else _now()
|
|
stamps = list((context.restarts if context else []) or [])
|
|
return [stamp for stamp in stamps if current - float(stamp) <= WINDOW_SECONDS]
|
|
|
|
|
|
def check_loop_guard(context: Optional[RestartContext], *, now: Optional[float] = None) -> None:
|
|
"""Refuse to restart if we've already done it too many times recently."""
|
|
stamps = recent_restarts(context, now=now)
|
|
if len(stamps) >= MAX_RESTARTS:
|
|
raise RestartError(
|
|
f"refusing: {len(stamps)} self-restarts in the last "
|
|
f"{int(WINDOW_SECONDS / 60)} minutes. Something is looping — fix the "
|
|
"cause, or wait for the window to clear before trying again."
|
|
)
|
|
|
|
|
|
def preflight(
|
|
repo: Optional[Path] = None,
|
|
run: Optional[Callable[..., Any]] = None,
|
|
timeout: float = 120.0,
|
|
) -> None:
|
|
"""Import the current source in a subprocess; raise if it's broken.
|
|
|
|
This process has the *old* modules loaded, so importing in-process would
|
|
happily succeed on a file that no longer parses. Mirrors
|
|
`updater._smoke_test`, and exists for the same reason: never hand the
|
|
session to code that can't start."""
|
|
runner = run or subprocess.run
|
|
root = Path(repo or config.HERE)
|
|
try:
|
|
completed = runner(
|
|
[sys.executable, "-c", _PREFLIGHT_IMPORT],
|
|
cwd=str(root), capture_output=True, text=True, timeout=timeout,
|
|
env={**os.environ, "QT_QPA_PLATFORM": "offscreen"}, # no display needed to import
|
|
)
|
|
except Exception as exc: # subprocess itself failed to run
|
|
raise RestartError(f"couldn't run the preflight import check: {exc}") from exc
|
|
if completed.returncode != 0:
|
|
detail = (completed.stderr or completed.stdout or "").strip()
|
|
raise RestartError(
|
|
"the current code does not import, so restarting would leave you with "
|
|
f"nothing running. Fix this first:\n{detail[-800:]}"
|
|
)
|
|
|
|
|
|
def arm(
|
|
reason: str,
|
|
*,
|
|
verify: str = "",
|
|
version: str = "",
|
|
session: str = "",
|
|
recent: Optional[list] = None,
|
|
path: Optional[Path] = None,
|
|
now: Optional[float] = None,
|
|
) -> RestartContext:
|
|
"""Record why we're about to die, carrying the restart history forward."""
|
|
current = now if now is not None else _now()
|
|
previous = load(path)
|
|
context = RestartContext(
|
|
reason=" ".join(str(reason or "").split())[:400],
|
|
verify=" ".join(str(verify or "").split())[:400],
|
|
armed_at=current,
|
|
version=str(version or ""),
|
|
session=str(session or ""),
|
|
recent=list(recent or [])[-6:],
|
|
restarts=recent_restarts(previous, now=current) + [current],
|
|
)
|
|
save(context, path)
|
|
return context
|
|
|
|
|
|
def report(
|
|
context: RestartContext,
|
|
*,
|
|
version: str = "",
|
|
now: Optional[float] = None,
|
|
) -> str:
|
|
"""The message the new process sends the server on the way up.
|
|
|
|
Phrased as Bolt reporting to himself, because that is what it is: the
|
|
server sees it as an ordinary turn, and the reply comes back through the
|
|
normal pipeline — which is what lets "restart and check X" finish as a
|
|
sentence spoken out loud."""
|
|
current = now if now is not None else _now()
|
|
took = max(0.0, current - float(context.armed_at or current))
|
|
lines = [
|
|
"[pet self-restart] I restarted myself and I'm back up.",
|
|
f"- reason: {context.reason or 'not recorded'}",
|
|
f"- took: {took:.1f}s",
|
|
f"- version now running: {version or 'unknown'}"
|
|
+ (f" (was {context.version})" if context.version and context.version != version else ""),
|
|
]
|
|
if context.verify:
|
|
lines.append(f"- you wanted to check: {context.verify}")
|
|
if context.recent:
|
|
lines.append("- what we were doing before: " + " | ".join(str(x)[:120] for x in context.recent))
|
|
lines.append(
|
|
"The new code is loaded and running. If you wanted to verify something, "
|
|
"check it now (filectl to read, command to test) and tell the user what you found."
|
|
)
|
|
return "\n".join(lines)
|