Files
themajesticmagician b121bbba17 Multi-monitor jumps, screen OCR, and generated sprite art
petctl gains screen verbs: `jump` (1-based number, name, next/prev/
primary/other, or a direction resolved from real geometry), `monitors`,
and `read` for OCR of a monitor's contents.

- monitors.py: pure layout model + jump-target resolution. The monitor
  list is published by PetWindow from QGuiApplication.screens() over a
  queued signal, so the controller and window agree on what "monitor 2"
  means; xrandr and Qt order screens differently on the same machine.
- screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR).
  Nothing captures unless the server asks, and the text rides back up
  the tool-result relay so Bolt can read a screen mid-turn. Both deps
  optional, soft-failing with a reason. SCREEN_TEXT=false removes it.
- Query verbs are answered in controller._handle_command rather than
  pet_actions.describe(), because their output is the point.
- scripts/generate_bolt_sprites.py draws every frame; walk/ is a
  side-view cycle stepped by distance travelled, not by the animation
  timer, so the planted paw tracks the window exactly. sprite.py loads
  it via EXTRA_ANIMATIONS keyed by name, with has() so callers can
  decline a placeholder blob.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-28 16:17:40 -06:00

214 lines
7.4 KiB
Python

"""Which screens exist, and which one the pet is standing on.
Deliberately free of Qt *and* of any subprocess probing: the monitor list is
published by the UI (`ui/pet_window.py` builds it from
`QGuiApplication.screens()`) and handed to the controller over a queued
signal, the same way every other UI↔controller message travels.
That indirection is the whole point. The pet has to agree with itself about
what "monitor 2" means — if the controller enumerated screens with `xrandr`
while the window jumped using Qt's screen list, the two orderings could
disagree and Bolt would announce one screen and land on another. Making Qt the
single source of truth removes that class of bug, and leaves everything here
pure enough to unit test without a display.
Indices are **1-based in every string a human or the model ever sees**, and
0-based in the list itself. `Monitor.index` is the 0-based one; `.number` is
what gets printed.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Iterable, Optional
# Directional specs understood by resolve(), mapped to a (dx, dy) heading.
_DIRECTIONS = {
"left": (-1, 0),
"right": (1, 0),
"up": (0, -1),
"above": (0, -1),
"down": (0, 1),
"below": (0, 1),
}
@dataclass(frozen=True)
class Monitor:
"""One screen, in the global desktop coordinate space."""
index: int # 0-based position in the published list
name: str
x: int
y: int
width: int
height: int
primary: bool = False
@property
def number(self) -> int:
"""1-based, for anything a person or the model reads."""
return self.index + 1
@property
def right(self) -> int:
return self.x + self.width
@property
def bottom(self) -> int:
return self.y + self.height
@property
def center(self) -> tuple[int, int]:
return self.x + self.width // 2, self.y + self.height // 2
def contains(self, x: int, y: int) -> bool:
return self.x <= x < self.right and self.y <= y < self.bottom
@property
def label(self) -> str:
bits = f"{self.number}: {self.name} {self.width}x{self.height}"
return bits + " (primary)" if self.primary else bits
def monitor_containing(
monitors: Iterable[Monitor], x: int, y: int
) -> Optional[Monitor]:
"""The screen holding point (x, y), or None if it's off every screen."""
for monitor in monitors:
if monitor.contains(x, y):
return monitor
return None
def nearest_monitor(monitors: Iterable[Monitor], x: int, y: int) -> Optional[Monitor]:
"""Screen whose centre is closest to (x, y) — the fallback when a point
lands in the dead space between mismatched screens."""
monitors = list(monitors)
if not monitors:
return None
return min(
monitors,
key=lambda m: (m.center[0] - x) ** 2 + (m.center[1] - y) ** 2,
)
def resolve(
monitors: list[Monitor], spec: str, current: Optional[int] = None
) -> Monitor:
"""Turn a `petctl jump` target into a screen.
Accepts a 1-based number, a name (case-insensitive substring, so "hdmi"
finds "HDMI-0"), `next`/`prev`, `primary`, `other`, or a direction
(`left`/`right`/`up`/`down`) relative to *current*. Raises ValueError with
a message meant to be read by the model, since it goes back as tool
output.
"""
if not monitors:
raise ValueError("no monitors have been reported yet")
spec = (spec or "").strip().lower()
if not spec:
raise ValueError("jump needs a target monitor")
count = len(monitors)
if current is None or not (0 <= current < count):
current = next((m.index for m in monitors if m.primary), 0)
if spec.isdigit():
number = int(spec)
if not (1 <= number <= count):
raise ValueError(
f"there is no monitor {number}; you have {count} "
f"(1-{count})"
)
return monitors[number - 1]
if spec in ("next", "forward"):
return monitors[(current + 1) % count]
if spec in ("prev", "previous", "back"):
return monitors[(current - 1) % count]
if spec == "primary":
return next((m for m in monitors if m.primary), monitors[0])
if spec == "other":
# With two screens "the other one" is unambiguous; with more it's just
# the next one round, which is at least always a *different* screen.
return monitors[(current + 1) % count]
if spec == "random":
# Deterministic-free choice is the caller's business; pick the screen
# furthest from the current one so "random" always visibly moves.
here = monitors[current].center
return max(
monitors,
key=lambda m: (m.center[0] - here[0]) ** 2 + (m.center[1] - here[1]) ** 2,
)
if spec in _DIRECTIONS:
dx, dy = _DIRECTIONS[spec]
here = monitors[current].center
candidates = []
for monitor in monitors:
if monitor.index == current:
continue
ox, oy = monitor.center
along = (ox - here[0]) * dx + (oy - here[1]) * dy
if along <= 0:
continue # not in that direction at all
drift = abs((ox - here[0]) * dy + (oy - here[1]) * dx)
candidates.append((drift, along, monitor))
if not candidates:
raise ValueError(
f"there's no monitor to the {spec} of monitor "
f"{monitors[current].number}"
)
# Prefer the best-aligned screen, then the closest of those.
candidates.sort(key=lambda item: (item[0], item[1]))
return candidates[0][2]
matches = [m for m in monitors if spec in m.name.lower()]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
raise ValueError(
f"{spec!r} matches several monitors: "
+ ", ".join(m.label for m in matches)
)
raise ValueError(
f"unknown monitor {spec!r}; you have: "
+ "; ".join(m.label for m in monitors)
+ " — or use next/prev/primary/left/right/up/down"
)
def describe(monitors: list[Monitor], current: Optional[int] = None) -> str:
"""Full listing, used as `petctl monitors` output."""
if not monitors:
return "[pet] no monitor information available"
lines = [f"[pet] {len(monitors)} monitor(s):"]
for monitor in monitors:
here = " <- Bolt is here" if monitor.index == current else ""
lines.append(
f" {monitor.label} at +{monitor.x}+{monitor.y}{here}"
)
return "\n".join(lines)
def summary(monitors: list[Monitor], current: Optional[int] = None) -> Optional[str]:
"""One-line version tacked onto each utterance — short on purpose, since
it rides along with every single thing you say."""
if not monitors:
return None
parts = ", ".join(f"{m.number}) {m.name} {m.width}x{m.height}" for m in monitors)
line = f"{len(monitors)} monitors: {parts}"
if current is not None and 0 <= current < len(monitors):
line += f"; Bolt is on {monitors[current].number}"
return line
def annotate(text: str, monitors: list[Monitor], current: Optional[int] = None) -> str:
"""Attach the screen summary as a separate aside, matching the style of
screen_context.annotate() so the model can ignore it when irrelevant."""
text = (text or "").strip()
line = summary(monitors, current)
if not text or not line:
return text
return f"{text}\n\n[{line}]"