b121bbba17
petctl gains screen verbs: `jump` (1-based number, name, next/prev/ primary/other, or a direction resolved from real geometry), `monitors`, and `read` for OCR of a monitor's contents. - monitors.py: pure layout model + jump-target resolution. The monitor list is published by PetWindow from QGuiApplication.screens() over a queued signal, so the controller and window agree on what "monitor 2" means; xrandr and Qt order screens differently on the same machine. - screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR). Nothing captures unless the server asks, and the text rides back up the tool-result relay so Bolt can read a screen mid-turn. Both deps optional, soft-failing with a reason. SCREEN_TEXT=false removes it. - Query verbs are answered in controller._handle_command rather than pet_actions.describe(), because their output is the point. - scripts/generate_bolt_sprites.py draws every frame; walk/ is a side-view cycle stepped by distance travelled, not by the animation timer, so the planted paw tracks the window exactly. sprite.py loads it via EXTRA_ANIMATIONS keyed by name, with has() so callers can decline a placeholder blob. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
214 lines
7.4 KiB
Python
214 lines
7.4 KiB
Python
"""Which screens exist, and which one the pet is standing on.
|
|
|
|
Deliberately free of Qt *and* of any subprocess probing: the monitor list is
|
|
published by the UI (`ui/pet_window.py` builds it from
|
|
`QGuiApplication.screens()`) and handed to the controller over a queued
|
|
signal, the same way every other UI↔controller message travels.
|
|
|
|
That indirection is the whole point. The pet has to agree with itself about
|
|
what "monitor 2" means — if the controller enumerated screens with `xrandr`
|
|
while the window jumped using Qt's screen list, the two orderings could
|
|
disagree and Bolt would announce one screen and land on another. Making Qt the
|
|
single source of truth removes that class of bug, and leaves everything here
|
|
pure enough to unit test without a display.
|
|
|
|
Indices are **1-based in every string a human or the model ever sees**, and
|
|
0-based in the list itself. `Monitor.index` is the 0-based one; `.number` is
|
|
what gets printed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Iterable, Optional
|
|
|
|
# Directional specs understood by resolve(), mapped to a (dx, dy) heading.
|
|
_DIRECTIONS = {
|
|
"left": (-1, 0),
|
|
"right": (1, 0),
|
|
"up": (0, -1),
|
|
"above": (0, -1),
|
|
"down": (0, 1),
|
|
"below": (0, 1),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Monitor:
|
|
"""One screen, in the global desktop coordinate space."""
|
|
|
|
index: int # 0-based position in the published list
|
|
name: str
|
|
x: int
|
|
y: int
|
|
width: int
|
|
height: int
|
|
primary: bool = False
|
|
|
|
@property
|
|
def number(self) -> int:
|
|
"""1-based, for anything a person or the model reads."""
|
|
return self.index + 1
|
|
|
|
@property
|
|
def right(self) -> int:
|
|
return self.x + self.width
|
|
|
|
@property
|
|
def bottom(self) -> int:
|
|
return self.y + self.height
|
|
|
|
@property
|
|
def center(self) -> tuple[int, int]:
|
|
return self.x + self.width // 2, self.y + self.height // 2
|
|
|
|
def contains(self, x: int, y: int) -> bool:
|
|
return self.x <= x < self.right and self.y <= y < self.bottom
|
|
|
|
@property
|
|
def label(self) -> str:
|
|
bits = f"{self.number}: {self.name} {self.width}x{self.height}"
|
|
return bits + " (primary)" if self.primary else bits
|
|
|
|
|
|
def monitor_containing(
|
|
monitors: Iterable[Monitor], x: int, y: int
|
|
) -> Optional[Monitor]:
|
|
"""The screen holding point (x, y), or None if it's off every screen."""
|
|
for monitor in monitors:
|
|
if monitor.contains(x, y):
|
|
return monitor
|
|
return None
|
|
|
|
|
|
def nearest_monitor(monitors: Iterable[Monitor], x: int, y: int) -> Optional[Monitor]:
|
|
"""Screen whose centre is closest to (x, y) — the fallback when a point
|
|
lands in the dead space between mismatched screens."""
|
|
monitors = list(monitors)
|
|
if not monitors:
|
|
return None
|
|
return min(
|
|
monitors,
|
|
key=lambda m: (m.center[0] - x) ** 2 + (m.center[1] - y) ** 2,
|
|
)
|
|
|
|
|
|
def resolve(
|
|
monitors: list[Monitor], spec: str, current: Optional[int] = None
|
|
) -> Monitor:
|
|
"""Turn a `petctl jump` target into a screen.
|
|
|
|
Accepts a 1-based number, a name (case-insensitive substring, so "hdmi"
|
|
finds "HDMI-0"), `next`/`prev`, `primary`, `other`, or a direction
|
|
(`left`/`right`/`up`/`down`) relative to *current*. Raises ValueError with
|
|
a message meant to be read by the model, since it goes back as tool
|
|
output.
|
|
"""
|
|
if not monitors:
|
|
raise ValueError("no monitors have been reported yet")
|
|
spec = (spec or "").strip().lower()
|
|
if not spec:
|
|
raise ValueError("jump needs a target monitor")
|
|
count = len(monitors)
|
|
if current is None or not (0 <= current < count):
|
|
current = next((m.index for m in monitors if m.primary), 0)
|
|
|
|
if spec.isdigit():
|
|
number = int(spec)
|
|
if not (1 <= number <= count):
|
|
raise ValueError(
|
|
f"there is no monitor {number}; you have {count} "
|
|
f"(1-{count})"
|
|
)
|
|
return monitors[number - 1]
|
|
|
|
if spec in ("next", "forward"):
|
|
return monitors[(current + 1) % count]
|
|
if spec in ("prev", "previous", "back"):
|
|
return monitors[(current - 1) % count]
|
|
if spec == "primary":
|
|
return next((m for m in monitors if m.primary), monitors[0])
|
|
if spec == "other":
|
|
# With two screens "the other one" is unambiguous; with more it's just
|
|
# the next one round, which is at least always a *different* screen.
|
|
return monitors[(current + 1) % count]
|
|
if spec == "random":
|
|
# Deterministic-free choice is the caller's business; pick the screen
|
|
# furthest from the current one so "random" always visibly moves.
|
|
here = monitors[current].center
|
|
return max(
|
|
monitors,
|
|
key=lambda m: (m.center[0] - here[0]) ** 2 + (m.center[1] - here[1]) ** 2,
|
|
)
|
|
|
|
if spec in _DIRECTIONS:
|
|
dx, dy = _DIRECTIONS[spec]
|
|
here = monitors[current].center
|
|
candidates = []
|
|
for monitor in monitors:
|
|
if monitor.index == current:
|
|
continue
|
|
ox, oy = monitor.center
|
|
along = (ox - here[0]) * dx + (oy - here[1]) * dy
|
|
if along <= 0:
|
|
continue # not in that direction at all
|
|
drift = abs((ox - here[0]) * dy + (oy - here[1]) * dx)
|
|
candidates.append((drift, along, monitor))
|
|
if not candidates:
|
|
raise ValueError(
|
|
f"there's no monitor to the {spec} of monitor "
|
|
f"{monitors[current].number}"
|
|
)
|
|
# Prefer the best-aligned screen, then the closest of those.
|
|
candidates.sort(key=lambda item: (item[0], item[1]))
|
|
return candidates[0][2]
|
|
|
|
matches = [m for m in monitors if spec in m.name.lower()]
|
|
if len(matches) == 1:
|
|
return matches[0]
|
|
if len(matches) > 1:
|
|
raise ValueError(
|
|
f"{spec!r} matches several monitors: "
|
|
+ ", ".join(m.label for m in matches)
|
|
)
|
|
raise ValueError(
|
|
f"unknown monitor {spec!r}; you have: "
|
|
+ "; ".join(m.label for m in monitors)
|
|
+ " — or use next/prev/primary/left/right/up/down"
|
|
)
|
|
|
|
|
|
def describe(monitors: list[Monitor], current: Optional[int] = None) -> str:
|
|
"""Full listing, used as `petctl monitors` output."""
|
|
if not monitors:
|
|
return "[pet] no monitor information available"
|
|
lines = [f"[pet] {len(monitors)} monitor(s):"]
|
|
for monitor in monitors:
|
|
here = " <- Bolt is here" if monitor.index == current else ""
|
|
lines.append(
|
|
f" {monitor.label} at +{monitor.x}+{monitor.y}{here}"
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def summary(monitors: list[Monitor], current: Optional[int] = None) -> Optional[str]:
|
|
"""One-line version tacked onto each utterance — short on purpose, since
|
|
it rides along with every single thing you say."""
|
|
if not monitors:
|
|
return None
|
|
parts = ", ".join(f"{m.number}) {m.name} {m.width}x{m.height}" for m in monitors)
|
|
line = f"{len(monitors)} monitors: {parts}"
|
|
if current is not None and 0 <= current < len(monitors):
|
|
line += f"; Bolt is on {monitors[current].number}"
|
|
return line
|
|
|
|
|
|
def annotate(text: str, monitors: list[Monitor], current: Optional[int] = None) -> str:
|
|
"""Attach the screen summary as a separate aside, matching the style of
|
|
screen_context.annotate() so the model can ignore it when irrelevant."""
|
|
text = (text or "").strip()
|
|
line = summary(monitors, current)
|
|
if not text or not line:
|
|
return text
|
|
return f"{text}\n\n[{line}]"
|