Multi-monitor jumps, screen OCR, and generated sprite art
petctl gains screen verbs: `jump` (1-based number, name, next/prev/ primary/other, or a direction resolved from real geometry), `monitors`, and `read` for OCR of a monitor's contents. - monitors.py: pure layout model + jump-target resolution. The monitor list is published by PetWindow from QGuiApplication.screens() over a queued signal, so the controller and window agree on what "monitor 2" means; xrandr and Qt order screens differently on the same machine. - screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR). Nothing captures unless the server asks, and the text rides back up the tool-result relay so Bolt can read a screen mid-turn. Both deps optional, soft-failing with a reason. SCREEN_TEXT=false removes it. - Query verbs are answered in controller._handle_command rather than pet_actions.describe(), because their output is the point. - scripts/generate_bolt_sprites.py draws every frame; walk/ is a side-view cycle stepped by distance travelled, not by the animation timer, so the planted paw tracks the window exactly. sprite.py loads it via EXTRA_ANIMATIONS keyed by name, with has() so callers can decline a placeholder blob. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,233 @@
|
||||
"""End-to-end wiring for the multi-monitor features: `petctl jump`, `petctl
|
||||
monitors`, `petctl read`, and the screen-layout note that rides along with
|
||||
each utterance.
|
||||
|
||||
Needs a QApplication (signals), so run with QT_QPA_PLATFORM=offscreen.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PySide6.QtWidgets import QApplication
|
||||
|
||||
from bolt_pet import controller as controller_mod
|
||||
from bolt_pet import pet_actions
|
||||
from bolt_pet.monitors import Monitor
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
_app = QApplication.instance() or QApplication(["test"])
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_screen_probes(monkeypatch):
|
||||
monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False)
|
||||
monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ctrl():
|
||||
controller = controller_mod.PetController()
|
||||
controller.set_monitors(
|
||||
[
|
||||
Monitor(0, "HDMI-0", 0, 0, 1920, 1080),
|
||||
Monitor(1, "HDMI-1", 1920, 0, 1920, 1080),
|
||||
Monitor(2, "DP-0", 0, 1080, 1920, 1080, primary=True),
|
||||
]
|
||||
)
|
||||
controller.set_pet_monitor(0)
|
||||
return controller
|
||||
|
||||
|
||||
def _capture(signal):
|
||||
events = []
|
||||
signal.connect(lambda *a: events.append(a[0] if len(a) == 1 else a))
|
||||
return events
|
||||
|
||||
|
||||
# ── parsing ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_jump_parses_without_validating_the_target():
|
||||
"""Which monitors exist is a runtime fact, so the pure parser passes the
|
||||
spec through and monitors.resolve() judges it later."""
|
||||
assert pet_actions.parse("petctl jump 2") == {"action": "jump", "target": "2"}
|
||||
assert pet_actions.parse("petctl jump next") == {"action": "jump", "target": "next"}
|
||||
assert pet_actions.parse("petctl monitor left") == {"action": "jump", "target": "left"}
|
||||
assert pet_actions.parse("petctl screen HDMI-1") == {"action": "jump", "target": "HDMI-1"}
|
||||
# a nonsense target is still parsed — it fails at resolve time, with a
|
||||
# message listing the real monitors
|
||||
assert pet_actions.parse("petctl jump sideways") == {
|
||||
"action": "jump", "target": "sideways",
|
||||
}
|
||||
|
||||
|
||||
def test_jump_needs_a_target():
|
||||
with pytest.raises(pet_actions.ActionError, match="needs a monitor"):
|
||||
pet_actions.parse("petctl jump")
|
||||
|
||||
|
||||
def test_read_defaults_to_the_current_screen():
|
||||
assert pet_actions.parse("petctl read") == {"action": "read", "target": "here"}
|
||||
assert pet_actions.parse("petctl read all") == {"action": "read", "target": "all"}
|
||||
assert pet_actions.parse("petctl look 2") == {"action": "read", "target": "2"}
|
||||
assert pet_actions.parse("petctl see here") == {"action": "read", "target": "here"}
|
||||
|
||||
|
||||
def test_monitors_verb():
|
||||
for spelling in ("monitors", "screens", "displays"):
|
||||
assert pet_actions.parse(f"petctl {spelling}") == {"action": "monitors"}
|
||||
|
||||
|
||||
def test_help_mentions_the_new_verbs():
|
||||
assert "petctl jump" in pet_actions.HELP
|
||||
assert "petctl read" in pet_actions.HELP
|
||||
assert "petctl monitors" in pet_actions.HELP
|
||||
|
||||
|
||||
# ── jump ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_jump_resolves_to_an_index_before_reaching_the_window(ctrl):
|
||||
"""The controller resolves and emits a concrete index, so the window can't
|
||||
re-resolve the spec against a different screen ordering."""
|
||||
actions = _capture(ctrl.action)
|
||||
out = ctrl._handle_command("petctl jump next")
|
||||
assert actions == [{"action": "jump", "monitor": 1}]
|
||||
assert "monitor 2: HDMI-1" in out
|
||||
|
||||
|
||||
def test_jump_by_direction_uses_the_published_layout(ctrl):
|
||||
actions = _capture(ctrl.action)
|
||||
ctrl._handle_command("petctl jump down")
|
||||
assert actions == [{"action": "jump", "monitor": 2}]
|
||||
|
||||
|
||||
def test_jump_tracks_where_the_pet_actually_is(ctrl):
|
||||
ctrl.set_pet_monitor(1)
|
||||
actions = _capture(ctrl.action)
|
||||
ctrl._handle_command("petctl jump left")
|
||||
assert actions == [{"action": "jump", "monitor": 0}]
|
||||
|
||||
|
||||
def test_jump_to_a_nonexistent_monitor_reports_back_and_moves_nothing(ctrl):
|
||||
actions = _capture(ctrl.action)
|
||||
out = ctrl._handle_command("petctl jump 7")
|
||||
assert actions == []
|
||||
assert "no monitor 7" in out
|
||||
assert "you have 3" in out
|
||||
|
||||
|
||||
def test_jump_never_reaches_the_shell(monkeypatch, ctrl):
|
||||
ran = []
|
||||
monkeypatch.setattr(controller_mod.server_client, "run_local_command", ran.append)
|
||||
ctrl._handle_command("petctl jump 2")
|
||||
ctrl._handle_command("petctl monitors")
|
||||
ctrl._handle_command("petctl read")
|
||||
assert ran == []
|
||||
|
||||
|
||||
# ── monitors ────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_monitors_lists_the_layout_and_where_the_pet_is(ctrl):
|
||||
ctrl.set_pet_monitor(2)
|
||||
out = ctrl._handle_command("petctl monitors")
|
||||
assert "3 monitor(s)" in out
|
||||
assert "HDMI-0" in out and "DP-0" in out
|
||||
assert out.count("Bolt is here") == 1
|
||||
|
||||
|
||||
def test_monitors_before_the_ui_has_published_anything():
|
||||
fresh = controller_mod.PetController()
|
||||
assert "no monitor information" in fresh._handle_command("petctl monitors")
|
||||
|
||||
|
||||
# ── read ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_read_here_uses_the_pets_own_screen(monkeypatch, ctrl):
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitor",
|
||||
lambda monitor, limit: seen.append(monitor.number) or "text",
|
||||
)
|
||||
ctrl.set_pet_monitor(1)
|
||||
assert ctrl._handle_command("petctl read") == "text"
|
||||
assert seen == [2]
|
||||
|
||||
|
||||
def test_read_a_named_screen(monkeypatch, ctrl):
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitor",
|
||||
lambda monitor, limit: seen.append(monitor.name) or "text",
|
||||
)
|
||||
ctrl._handle_command("petctl read DP-0")
|
||||
assert seen == ["DP-0"]
|
||||
|
||||
|
||||
def test_read_all_goes_through_the_multi_screen_path(monkeypatch, ctrl):
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitors",
|
||||
lambda monitors, limit: seen.append(len(monitors)) or "everything",
|
||||
)
|
||||
assert ctrl._handle_command("petctl read all") == "everything"
|
||||
assert seen == [3]
|
||||
|
||||
|
||||
def test_read_an_unknown_screen_explains_rather_than_raising(ctrl):
|
||||
out = ctrl._handle_command("petctl read 9")
|
||||
assert out.startswith("[pet]")
|
||||
assert "no monitor 9" in out
|
||||
|
||||
|
||||
def test_read_respects_the_kill_switch(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "SCREEN_TEXT", False)
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitor",
|
||||
lambda *a, **k: called.append(1) or "text",
|
||||
)
|
||||
out = ctrl._handle_command("petctl read")
|
||||
assert called == []
|
||||
assert "disabled" in out and "SCREEN_TEXT" in out
|
||||
|
||||
|
||||
def test_read_passes_the_character_cap_through(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "SCREEN_TEXT_MAX_CHARS", 123)
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitor",
|
||||
lambda monitor, limit: seen.append(limit) or "text",
|
||||
)
|
||||
ctrl._handle_command("petctl read")
|
||||
assert seen == [123]
|
||||
|
||||
|
||||
# ── per-turn context ────────────────────────────────────────────────────────
|
||||
|
||||
def test_layout_rides_along_with_each_utterance(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", True)
|
||||
out = ctrl._with_context("what's on the other screen?")
|
||||
assert out.startswith("what's on the other screen?")
|
||||
assert "3 monitors:" in out
|
||||
assert "Bolt is on 1" in out
|
||||
|
||||
|
||||
def test_layout_context_can_be_switched_off(monkeypatch, ctrl):
|
||||
monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", False)
|
||||
assert ctrl._with_context("hello") == "hello"
|
||||
|
||||
|
||||
def test_screen_text_never_rides_along_automatically(monkeypatch, ctrl):
|
||||
"""The layout is free; the *contents* cost an OCR pass and a lot of
|
||||
privacy, so they only ever move on an explicit petctl read."""
|
||||
monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", True)
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitor", lambda *a, **k: called.append(1)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller_mod.screen_text, "read_monitors", lambda *a, **k: called.append(1)
|
||||
)
|
||||
ctrl._with_context("hello")
|
||||
assert called == []
|
||||
Reference in New Issue
Block a user