themajesticmagician
b121bbba17
petctl gains screen verbs: `jump` (1-based number, name, next/prev/ primary/other, or a direction resolved from real geometry), `monitors`, and `read` for OCR of a monitor's contents. - monitors.py: pure layout model + jump-target resolution. The monitor list is published by PetWindow from QGuiApplication.screens() over a queued signal, so the controller and window agree on what "monitor 2" means; xrandr and Qt order screens differently on the same machine. - screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR). Nothing captures unless the server asks, and the text rides back up the tool-result relay so Bolt can read a screen mid-turn. Both deps optional, soft-failing with a reason. SCREEN_TEXT=false removes it. - Query verbs are answered in controller._handle_command rather than pet_actions.describe(), because their output is the point. - scripts/generate_bolt_sprites.py draws every frame; walk/ is a side-view cycle stepped by distance travelled, not by the animation timer, so the planted paw tracks the window exactly. sprite.py loads it via EXTRA_ANIMATIONS keyed by name, with has() so callers can decline a placeholder blob. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
169 lines
6.3 KiB
Python
169 lines
6.3 KiB
Python
"""OCR plumbing for `petctl read` — engine selection and output cleanup.
|
|
|
|
Only the pure half is covered, per the testing conventions: capture and the
|
|
OCR call itself need a real screen and a real engine. Engine probes are
|
|
injected so these pass on a machine with a different set installed (or none).
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from bolt_pet import screen_text
|
|
from bolt_pet.monitors import Monitor
|
|
|
|
|
|
def probes(modules=(), binaries=()):
|
|
return (lambda name: name in modules), (
|
|
lambda name: f"/usr/bin/{name}" if name in binaries else None
|
|
)
|
|
|
|
|
|
def test_prefers_tesseract_when_fully_installed():
|
|
has_module, which = probes({"pytesseract"}, {"tesseract"})
|
|
assert screen_text.resolve_engine(has_module, which) == ("pytesseract", "")
|
|
|
|
|
|
def test_falls_back_to_rapidocr_when_tesseract_binary_is_absent():
|
|
has_module, which = probes({"pytesseract", "rapidocr_onnxruntime"}, set())
|
|
engine, reason = screen_text.resolve_engine(has_module, which)
|
|
assert engine == "rapidocr"
|
|
assert reason == ""
|
|
|
|
|
|
def test_pytesseract_without_the_binary_says_which_half_is_missing():
|
|
"""The commonest broken setup: `pip install pytesseract` and stop, not
|
|
realising the actual engine is a system package."""
|
|
has_module, which = probes({"pytesseract"}, set())
|
|
engine, reason = screen_text.resolve_engine(has_module, which)
|
|
assert engine is None
|
|
assert "tesseract binary" in reason
|
|
assert "apt install tesseract-ocr" in reason
|
|
|
|
|
|
def test_nothing_installed_explains_how_to_fix_it():
|
|
has_module, which = probes(set(), set())
|
|
engine, reason = screen_text.resolve_engine(has_module, which)
|
|
assert engine is None
|
|
assert "pip install" in reason
|
|
|
|
|
|
def test_capture_availability_follows_mss():
|
|
assert screen_text.capture_available(lambda name: name == "mss")
|
|
assert not screen_text.capture_available(lambda name: False)
|
|
|
|
|
|
def test_clean_drops_ocr_noise_and_blank_runs():
|
|
raw = "Firefox\n\n\n |\n .\nBuild failed\n~\n"
|
|
assert screen_text.clean_ocr_text(raw) == "Firefox\nBuild failed"
|
|
|
|
|
|
def test_clean_collapses_whitespace_but_keeps_line_structure():
|
|
raw = " File edit view \nline\ttwo "
|
|
assert screen_text.clean_ocr_text(raw) == "File edit view\nline two"
|
|
|
|
|
|
def test_clean_drops_consecutive_duplicates_only():
|
|
raw = "Terminal\nTerminal\nEditor\nTerminal"
|
|
assert screen_text.clean_ocr_text(raw) == "Terminal\nEditor\nTerminal"
|
|
|
|
|
|
def test_clean_keeps_short_but_real_tokens():
|
|
# two alphanumerics is the bar — "ok" and "42" survive, "-" doesn't
|
|
assert screen_text.clean_ocr_text("ok\n-\n42") == "ok\n42"
|
|
|
|
|
|
def test_clean_truncates_and_says_so():
|
|
out = screen_text.clean_ocr_text("word " * 500, max_chars=100)
|
|
assert out.endswith("[truncated]")
|
|
# the cap applies to the text, before the marker is appended
|
|
assert len(out.split("\n[truncated]")[0]) <= 100
|
|
|
|
|
|
def test_clean_handles_empty_input():
|
|
assert screen_text.clean_ocr_text("") == ""
|
|
assert screen_text.clean_ocr_text(None) == ""
|
|
|
|
|
|
def test_format_reading_names_the_monitor():
|
|
monitor = Monitor(1, "HDMI-1", 1920, 0, 1920, 1080)
|
|
out = screen_text.format_reading(monitor, "Build failed")
|
|
assert "monitor 2 (HDMI-1)" in out
|
|
assert out.endswith("Build failed")
|
|
|
|
|
|
def test_format_reading_when_nothing_was_recognised():
|
|
monitor = Monitor(0, "DP-0", 0, 0, 1920, 1080)
|
|
assert "no text recognised" in screen_text.format_reading(monitor, " ")
|
|
|
|
|
|
def test_read_monitor_never_raises_without_an_engine(monkeypatch):
|
|
"""Its return value goes back to the server as command output, so every
|
|
failure has to come back as a sentence rather than an exception."""
|
|
monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True)
|
|
monkeypatch.setattr(
|
|
screen_text, "resolve_engine", lambda *a, **k: (None, "no engine here")
|
|
)
|
|
out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080))
|
|
assert out.startswith("[pet]")
|
|
assert "no engine here" in out
|
|
|
|
|
|
def test_read_monitor_reports_a_failed_capture(monkeypatch):
|
|
monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True)
|
|
monkeypatch.setattr(screen_text, "resolve_engine", lambda *a, **k: ("pytesseract", ""))
|
|
monkeypatch.setattr(screen_text, "capture", lambda monitor: None)
|
|
out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080))
|
|
assert "couldn't capture" in out and "Wayland" in out
|
|
|
|
|
|
def test_read_monitor_survives_an_exploding_engine(monkeypatch):
|
|
monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True)
|
|
monkeypatch.setattr(screen_text, "resolve_engine", lambda *a, **k: ("pytesseract", ""))
|
|
monkeypatch.setattr(screen_text, "capture", lambda monitor: object())
|
|
monkeypatch.setattr(
|
|
screen_text, "_ocr", lambda image, engine: (_ for _ in ()).throw(RuntimeError("boom"))
|
|
)
|
|
out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080))
|
|
assert "OCR failed" in out and "boom" in out
|
|
|
|
|
|
def test_read_monitors_splits_the_budget(monkeypatch):
|
|
seen = []
|
|
|
|
def fake_read(monitor, max_chars):
|
|
seen.append((monitor.number, max_chars))
|
|
return f"screen {monitor.number}"
|
|
|
|
monkeypatch.setattr(screen_text, "read_monitor", fake_read)
|
|
screens = [
|
|
Monitor(0, "A", 0, 0, 100, 100),
|
|
Monitor(1, "B", 100, 0, 100, 100),
|
|
Monitor(2, "C", 200, 0, 100, 100),
|
|
]
|
|
out = screen_text.read_monitors(screens, 3000)
|
|
assert [n for n, _ in seen] == [1, 2, 3]
|
|
assert all(limit == 1000 for _, limit in seen)
|
|
assert out.count("screen ") == 3
|
|
|
|
|
|
def test_read_monitors_keeps_a_floor_on_the_budget(monkeypatch):
|
|
monkeypatch.setattr(
|
|
screen_text, "read_monitor", lambda monitor, max_chars: str(max_chars)
|
|
)
|
|
screens = [Monitor(i, str(i), 0, 0, 10, 10) for i in range(20)]
|
|
# 100/20 would be 5 characters per screen, which is useless — floor wins
|
|
assert "400" in screen_text.read_monitors(screens, 100)
|
|
|
|
|
|
def test_read_monitors_with_one_screen_uses_the_whole_budget(monkeypatch):
|
|
monkeypatch.setattr(
|
|
screen_text, "read_monitor", lambda monitor, max_chars: str(max_chars)
|
|
)
|
|
assert screen_text.read_monitors([Monitor(0, "A", 0, 0, 10, 10)], 4000) == "4000"
|
|
|
|
|
|
def test_read_monitors_with_no_screens():
|
|
assert "no monitor information" in screen_text.read_monitors([], 4000)
|