diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 7ab6472..b11fe32 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -30,7 +30,17 @@ "Bash(git push *)", "Bash(git remote *)", "Bash(grep -v '^$')", - "Bash(.venv/bin/pip install *)" + "Bash(.venv/bin/pip install *)", + "Bash(QT_QPA_PLATFORM=offscreen .venv/bin/python *)", + "Bash(python3 *)", + "Bash(.venv/bin/pytest tests/test_monitors.py -q)", + "Bash(QT_QPA_PLATFORM=offscreen .venv/bin/pytest tests/test_pet_window_features.py -q)", + "Bash(git fetch *)", + "Bash(git switch *)", + "Bash(git add *)", + "Bash(git merge *)", + "Bash(echo \"=== EXIT: $? ===\")", + "Bash(git commit *)" ] } } diff --git a/.env.example b/.env.example index 1cfd165..b40c884 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,30 @@ ELEVENLABS_VOICE_ID= # error?" has a referent. Text only — no screenshots leave the machine. #SCREEN_CONTEXT=true +# ── Monitors (optional) ───────────────────────────────────────────────────── +# Tacks a one-line summary of your screen layout onto each utterance (how +# many, their sizes, which one the pet is standing on) so Bolt can decide to +# `petctl jump 2` without asking what you've got plugged in. Costs nothing — +# the list comes from the UI, nothing is probed per turn. +#MONITOR_CONTEXT=true + +# ── Screen text / OCR (optional) ──────────────────────────────────────────── +# Lets Bolt read what's actually on a monitor with `petctl read [n|here|all]` +# and use it in his reply. Pull-only — nothing is captured unless he asks, +# and every read is logged. +# +# Needs the extras from requirements.txt plus an OCR engine: +# pip install mss pytesseract && sudo apt install tesseract-ocr +# or, without sudo: +# pip install mss rapidocr-onnxruntime +# mss captures on X11/Windows/macOS but NOT Wayland. +# +# This sends the text of a whole screen to the server when used. That's not a +# new capability — the shell relay could already screenshot and OCR — but it's +# a far easier one to reach for. SCREEN_TEXT=false removes it entirely. +#SCREEN_TEXT=true +#SCREEN_TEXT_MAX_CHARS=4000 + # ── Quiet hours / do-not-disturb (optional) ──────────────────────────────── # Comma-separated HH:MM-HH:MM ranges; wrapping past midnight is fine. While # napping the pet dims, stops wandering and makes no proactive noise — the diff --git a/CLAUDE.md b/CLAUDE.md index 6d1e394..12f77c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,13 +11,18 @@ dependency** on the server repo; it's a standalone HTTP client configured via its own `.env`. Pipeline: `mic → openWakeWord ("thunderbolt", on-device) / push-to-talk / -click → record utterance → Deepgram STT → + active-window context → POST -/desk/converse → [server may relay a shell command to run on this machine, or -a `petctl` pseudo-command that moves/emotes the pet instead] → reply → +click → record utterance → Deepgram STT → + active-window + screen-layout +context → POST /desk/converse → [server may relay a shell command to run on +this machine, or a `petctl` pseudo-command that moves/emotes the pet, jumps it +to another monitor, or reads a screen's text back instead] → reply → ElevenLabs streaming TTS (or offline pyttsx3 fallback) → speakers`, with the pet sprite/speech bubble reflecting state throughout, and playback interruptible by talking over it (barge-in). +Because a relayed command's output goes back up the tool-result relay before +the final reply, a `petctl read` mid-turn means Bolt can look at a monitor and +then talk about what's on it in the same answer. + Side channels that let the pet act between turns: the heartbeat (proactive announcements), the desktop notification bridge, and autonomous wandering — all suppressed while it's napping (quiet hours / fullscreen DND). @@ -34,6 +39,9 @@ run.bat # Windows QT_QPA_PLATFORM=offscreen .venv/bin/pytest tests/ .venv/bin/pytest tests/test_state.py::test_happy_path_transitions # single test +# Redraw the pet's sprite frames (the committed PNGs are this script's output) +python scripts/generate_bolt_sprites.py # --out /tmp/x to preview first + # Convert a grid sprite sheet into the per-frame-PNG convention sprite.py expects python scripts/slice_spritesheet.py path/to/sheet.png assets/sprites/idle --cols 6 --rows 1 ``` @@ -90,17 +98,46 @@ logs a missing-config message and exits its thread instead of starting. accepts an injectable stream/model/protocol so tests don't need real audio hardware or a display. - **`pet_actions.py`** — `petctl` pseudo-commands (`petctl move top-left`, - `petctl emote wave`, `say`/`wander`/`nap`). The desk API has no "move the - pet" payload type and this repo can't change the server, so these ride the - existing shell-command relay: `controller._handle_command` parses them and - they never reach `subprocess`; anything else is a real shell command exactly - as before. Pure parsing; the UI half is `PetWindow.apply_action`. + `petctl emote wave`, `say`/`wander`/`nap`, plus the screen verbs + `jump`/`monitors`/`read`). The desk API has no "move the pet" payload type + and this repo can't change the server, so these ride the existing + shell-command relay: `controller._handle_command` parses them and they never + reach `subprocess`; anything else is a real shell command exactly as before. + Pure parsing; the UI half is `PetWindow.apply_action`. Note `jump`'s target + is *not* validated here — which monitors exist is a runtime fact this pure + module doesn't have, so the spec passes through to `monitors.resolve()`. + Query verbs (`monitors`, `read`) are answered in `_handle_command` rather + than by `pet_actions.describe()`, because their output *is* the point: it + goes back up the tool-result relay for Bolt to use in his reply. - **`screen_context.py`** — active-window title (xprop/xdotool, Win32, osascript) appended to each utterance via `context_for()`, plus `is_fullscreen_active()` for do-not-disturb. Text only — the desk API takes no images. Every probe is best-effort and returns None/False rather than raising; the parsing is split into pure functions that are tested without a display server. +- **`monitors.py`** — the screen layout, and resolving `petctl jump` targets + (a 1-based number, a name, `next`/`prev`/`primary`/`other`, or a direction + like `left`/`up` worked out from the actual geometry). Pure — no Qt, no + subprocess. The monitor list is *published by the UI* + (`PetWindow.publish_monitors` builds it from `QGuiApplication.screens()` and + emits it over a queued signal to `controller.set_monitors`), because the + controller and the window must agree on what "monitor 2" means: enumerating + with `xrandr` on one side and Qt's screen list on the other gives different + orderings on the same machine, and Bolt would announce one screen and land + on another. Qt is the single source of truth; `Monitor.index` is 0-based and + `.number` is the 1-based value used in every string a human or the model + sees. The controller resolves a jump to a concrete index *before* emitting + it, so the window can't re-resolve against a different list. +- **`screen_text.py`** — OCR, so Bolt can read what's on a monitor + (`petctl read [n|here|all]`). **Pull, not push**: nothing captures on its own + — the server has to ask, and the text goes back as that command's output. + That's deliberate; OCR of a 4K screen costs a second or two that would + otherwise be added to *every* utterance, and screen contents leaving the + machine should be a visible decision rather than a constant. Capture needs + `mss` (X11/Win32/macOS, **not** Wayland), recognition needs Tesseract or + RapidOCR; both are optional and soft-fail with a reason the way `hotkey.py` + does, and `read_monitor()` never raises because its return value is command + output. Engine selection takes injected probes so it's testable wherever. - **`quiet.py`** — quiet-hours spec parsing (`23:00-08:00`, wraps midnight, comma-separated). Napping suppresses *proactive* noise and wandering only; wake word / click / push-to-talk still work. @@ -162,6 +199,15 @@ logs a missing-config message and exits its thread instead of starting. target every `PET_WANDER_INTERVAL_SECONDS` (randomized), suppressed whenever the pet is non-IDLE, napping, dragged, or has a bubble up. A commanded `petctl move` overrides all of that except the drag. + - **the walk cycle** — while actually travelling, `_animation_key()` swaps + the state animation for the side-view `walk/` frames (not a `PetState` — + see the sprites README). It is stepped by *distance travelled* + (`_WALK_PIXELS_PER_FRAME`), never by the animation timer, so the planted + paw tracks backwards at exactly the speed the window moves forwards; + `_advance_frame` deliberately no-ops while walking so the two can't + double-step it. The art is drawn facing right and `_oriented()` mirrors it + (cached per frame) when heading left. No `walk/` art → falls back to the + old coded bob rather than a placeholder blob. - **emotes** — `emote_transform()` is pure maths (dx, dy, rotation, scale from a 0..1 progress) kept out of `paintEvent` so the curves are unit tested; every emote must return to the identity transform at progress 1.0 @@ -174,9 +220,14 @@ logs a missing-config message and exits its thread instead of starting. - **edge snapping** (`PET_EDGE_SNAP`) after a drag or a stroll, and **nap dimming** (`set_napping`). `sprite.py` loads `assets/sprites//*.png` (filename-sorted, looping — - currently Kenney's CC0 robot pack, see `assets/sprites/README.md`) and falls + the art is *generated* by `scripts/generate_bolt_sprites.py`, a Pillow + drawing of Bolt as a shepherd pup; edit the script and re-run it rather than + the committed PNGs, see `assets/sprites/README.md`) and falls back to a procedurally-drawn placeholder blob per state if a folder has no - frames; `tray.py` is the system tray menu (talk now / mute / nap / wander / + frames. It also loads `EXTRA_ANIMATIONS` — currently just `walk/` — keyed by + name rather than by `PetState`, with `has()` reporting whether a key is + backed by real art so callers can decline a placeholder instead of trotting + a blob across the desktop; `tray.py` is the system tray menu (talk now / mute / nap / wander / click-through / history / wake-word tuning / quit) — the pet window has no title bar or taskbar entry; `history_window.py` and `wake_tuner.py` are the two dialogs it opens. @@ -262,9 +313,23 @@ can't reach the user's PipeWire socket from a root session (raw ALSA devices reject the 16 kHz capture rate — `paInvalidSampleRate`), and every relayed command would run unconstrained. -Two newer features widen what leaves this machine, both switchable in `.env`: +Several features widen what leaves this machine, all switchable in `.env`: `SCREEN_CONTEXT` appends the focused window's *title* to each utterance -(titles often contain file paths, document names, or subject lines), and -`NOTIFICATION_BRIDGE` (off by default) forwards matching desktop -notifications to the server. Neither sends screenshots or notification -contents you haven't matched with `NOTIFICATION_FILTER`. +(titles often contain file paths, document names, or subject lines), +`MONITOR_CONTEXT` appends the screen layout (sizes and names only — no +contents), and `NOTIFICATION_BRIDGE` (off by default) forwards matching +desktop notifications to the server. None of those send screenshots or +notification contents you haven't matched with `NOTIFICATION_FILTER`. + +`SCREEN_TEXT` is the biggest of them: `petctl read` OCRs a whole monitor and +sends the recognised text to the server — everything visible, not just the +focused window. Two things keep it honest. It's **pull-only**: no capture +happens unless the server explicitly asks, so it can't leak in the background +the way a per-turn annotation would, and each read is logged. And it is +strictly *not* a new capability — the shell relay could already run a +screenshot tool and pipe it through OCR — it just makes a thing the trust +model already allowed reliable, bounded (`SCREEN_TEXT_MAX_CHARS`) and +visible. It is nonetheless far easier to reach for than the shell route, so +if that trade isn't one you want, `SCREEN_TEXT=false` removes it and +`petctl read` starts reporting that it's disabled. Capture is `mss`-based and +therefore silently unavailable on Wayland. diff --git a/bolt_pet/assets/sprites/README.md b/bolt_pet/assets/sprites/README.md index a77b241..3bea938 100644 --- a/bolt_pet/assets/sprites/README.md +++ b/bolt_pet/assets/sprites/README.md @@ -1,37 +1,84 @@ # Sprite assets -Art: [Kenney's Robot Pack](https://kenney.nl/assets/robot-pack) (CC0 — no -attribution required, credited here anyway), the green side-view robot. -Source pack lives at `~/Documents/kenney_robot-pack`; only the frames listed -below were copied in. +Art: Bolt himself — a cream shepherd pup with a slate cap, a lightning blaze +on his forehead and a bolt tag on his collar. The frames are **generated, not +hand-drawn**: `scripts/generate_bolt_sprites.py` draws every one of them with +Pillow and writes this folder. + +```bash +python scripts/generate_bolt_sprites.py # rewrite this folder +python scripts/generate_bolt_sprites.py --out /tmp/prev # preview elsewhere first +python scripts/generate_bolt_sprites.py --states idle # just one state +``` + +That means tweaking the art is editing code, not 24 PNGs: the palette is a +block of constants at the top of the script, the body/head/ear/tail shapes are +one function each in normalised 0..1 coordinates, and each state's animation is +a list of pose dicts in `frames_for()`. Everything is super-sampled 4x and +downscaled on save, because PIL's draw primitives have no antialiasing. + +**Regenerate after editing** — the PNGs here are committed, so a change to the +script alone doesn't move the pet. Convention the loader (`bolt_pet/ui/sprite.py`) expects: ``` assets/sprites/ - idle/ frame_00.png robot_greenBody (standing) - listening/ frame_00.png, frame_01.png robot_greenDrive1/2 (tracks rolling — "leaning in") - thinking/ frame_00.png, frame_01.png robot_greenDamage1/2 (flicker — "processing") - talking/ frame_00.png, frame_01.png robot_greenBody, robot_greenJump (bounce) - error/ frame_00.png robot_greenHurt + idle/ frame_00..07.png breathing, tail wag, blink on frame 06 + listening/ frame_00..03.png ears perked, head tilted in, collar tag lit, sound arcs + thinking/ frame_00..05.png eyes up, head cocked, cycling dots + talking/ frame_00..03.png mouth open/close with tongue, ears bouncing + error/ frame_00..01.png X eyes, ears drooped, red spark + walk/ frame_00..07.png side-view walk cycle (see below) ``` -- One subfolder per pet state (matches `bolt_pet.state.PetState`). +- One subfolder per pet state (matches `bolt_pet.state.PetState`), **plus + `walk/`**, which is not a state — see below. - Any `*.png` filenames work — they're played back in alphabetical-sort - order, looping, at `IDLE_ANIMATION_FPS` (see `.env`). -- Frames are scaled to fit within `PET_SIZE` (default 160px), keeping aspect - ratio, and centered in the (square) pet window — the source art here isn't - square, so don't assume it fills the frame edge-to-edge. + order, looping, at `IDLE_ANIMATION_FPS` (see `.env`). At the default 6fps + the 8-frame idle loop runs about 1.3s. +- Frames are square (320px, 2x the default `PET_SIZE` of 160) so they + downscale cleanly; the loader scales to fit `PET_SIZE` keeping aspect ratio + and centres them in the square pet window. - A state directory with no frames in it falls back to a small procedurally-drawn placeholder blob (see `_placeholder_frames` in `sprite.py`). +## The walk cycle + +`walk/` is the one animation that isn't a `PetState`. Walking is a property of +*movement* — orthogonal to whether he's idle, listening or talking — so it +stays out of the state machine and is keyed by name instead +(`sprite.EXTRA_ANIMATIONS`). `PetWindow` uses it whenever the pet is actually +travelling and falls back to the state animation the moment it stops. + +Three things about it are load-bearing if you redraw it: + +- **It's a side view, drawn facing right.** The other poses are a + front-facing sit, which is fine standing still but slides like a chess + piece when moving. `PetWindow._oriented()` mirrors the frames (cached) when + he walks left, so only the right-facing version exists on disk. +- **The cycle is advanced by distance travelled, not by the animation + timer** (`_WALK_PIXELS_PER_FRAME`, one frame per ~13px). That's what keeps + a planted paw tracking backwards at exactly the speed the window moves + forwards. Drive it off the clock and the feet skate whenever + `PET_WANDER_SPEED` doesn't happen to match `IDLE_ANIMATION_FPS`. If you + change the number of frames or the stride length in + `paw_position()`, retune that constant to match or he'll moonwalk. +- **The frames carry their own vertical bob**, so the window's own bob is + switched off while they're in use. Only the no-walk-art fallback still + bobs in code. + +Delete `walk/` and everything still runs — he reverts to sliding with a small +coded bob, which is what the pet did before the cycle existed. + ## Swapping in different art Replace any state's PNGs (same alphabetical-order-loops convention) to -change its look — no code changes needed. If your source is a single grid -spritesheet (rows/cols of frames in one PNG) rather than one-file-per-frame, -use `scripts/slice_spritesheet.py` to cut it into this folder-of-frames +change its look — no code changes needed, and nothing forces you to keep +using the generator. If your source is a single grid spritesheet (rows/cols +of frames in one PNG) rather than one-file-per-frame, use +`scripts/slice_spritesheet.py` to cut it into this folder-of-frames convention: ```bash diff --git a/bolt_pet/assets/sprites/error/frame_00.png b/bolt_pet/assets/sprites/error/frame_00.png index b0583aa..6cd4216 100644 Binary files a/bolt_pet/assets/sprites/error/frame_00.png and b/bolt_pet/assets/sprites/error/frame_00.png differ diff --git a/bolt_pet/assets/sprites/error/frame_01.png b/bolt_pet/assets/sprites/error/frame_01.png new file mode 100644 index 0000000..75096f7 Binary files /dev/null and b/bolt_pet/assets/sprites/error/frame_01.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_00.png b/bolt_pet/assets/sprites/idle/frame_00.png index 297722b..3b01320 100644 Binary files a/bolt_pet/assets/sprites/idle/frame_00.png and b/bolt_pet/assets/sprites/idle/frame_00.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_01.png b/bolt_pet/assets/sprites/idle/frame_01.png new file mode 100644 index 0000000..0d1cc60 Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_01.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_02.png b/bolt_pet/assets/sprites/idle/frame_02.png new file mode 100644 index 0000000..f7a055c Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_02.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_03.png b/bolt_pet/assets/sprites/idle/frame_03.png new file mode 100644 index 0000000..da58383 Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_03.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_04.png b/bolt_pet/assets/sprites/idle/frame_04.png new file mode 100644 index 0000000..3b01320 Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_04.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_05.png b/bolt_pet/assets/sprites/idle/frame_05.png new file mode 100644 index 0000000..55c6a70 Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_05.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_06.png b/bolt_pet/assets/sprites/idle/frame_06.png new file mode 100644 index 0000000..e21d5d2 Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_06.png differ diff --git a/bolt_pet/assets/sprites/idle/frame_07.png b/bolt_pet/assets/sprites/idle/frame_07.png new file mode 100644 index 0000000..ba3c27c Binary files /dev/null and b/bolt_pet/assets/sprites/idle/frame_07.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_00.png b/bolt_pet/assets/sprites/listening/frame_00.png index b460239..d4872c5 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_00.png and b/bolt_pet/assets/sprites/listening/frame_00.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_01.png b/bolt_pet/assets/sprites/listening/frame_01.png index bbf8077..93f44ee 100644 Binary files a/bolt_pet/assets/sprites/listening/frame_01.png and b/bolt_pet/assets/sprites/listening/frame_01.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_02.png b/bolt_pet/assets/sprites/listening/frame_02.png new file mode 100644 index 0000000..ee23890 Binary files /dev/null and b/bolt_pet/assets/sprites/listening/frame_02.png differ diff --git a/bolt_pet/assets/sprites/listening/frame_03.png b/bolt_pet/assets/sprites/listening/frame_03.png new file mode 100644 index 0000000..c2bbf76 Binary files /dev/null and b/bolt_pet/assets/sprites/listening/frame_03.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_00.png b/bolt_pet/assets/sprites/talking/frame_00.png index 297722b..8559181 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_00.png and b/bolt_pet/assets/sprites/talking/frame_00.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_01.png b/bolt_pet/assets/sprites/talking/frame_01.png index 356cfd0..818e23c 100644 Binary files a/bolt_pet/assets/sprites/talking/frame_01.png and b/bolt_pet/assets/sprites/talking/frame_01.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_02.png b/bolt_pet/assets/sprites/talking/frame_02.png new file mode 100644 index 0000000..2974be7 Binary files /dev/null and b/bolt_pet/assets/sprites/talking/frame_02.png differ diff --git a/bolt_pet/assets/sprites/talking/frame_03.png b/bolt_pet/assets/sprites/talking/frame_03.png new file mode 100644 index 0000000..8c930e4 Binary files /dev/null and b/bolt_pet/assets/sprites/talking/frame_03.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_00.png b/bolt_pet/assets/sprites/thinking/frame_00.png index bf6ac56..cf5da13 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_00.png and b/bolt_pet/assets/sprites/thinking/frame_00.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_01.png b/bolt_pet/assets/sprites/thinking/frame_01.png index 806d10e..a033ebe 100644 Binary files a/bolt_pet/assets/sprites/thinking/frame_01.png and b/bolt_pet/assets/sprites/thinking/frame_01.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_02.png b/bolt_pet/assets/sprites/thinking/frame_02.png new file mode 100644 index 0000000..5da2a87 Binary files /dev/null and b/bolt_pet/assets/sprites/thinking/frame_02.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_03.png b/bolt_pet/assets/sprites/thinking/frame_03.png new file mode 100644 index 0000000..986f326 Binary files /dev/null and b/bolt_pet/assets/sprites/thinking/frame_03.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_04.png b/bolt_pet/assets/sprites/thinking/frame_04.png new file mode 100644 index 0000000..0729e2e Binary files /dev/null and b/bolt_pet/assets/sprites/thinking/frame_04.png differ diff --git a/bolt_pet/assets/sprites/thinking/frame_05.png b/bolt_pet/assets/sprites/thinking/frame_05.png new file mode 100644 index 0000000..0729e2e Binary files /dev/null and b/bolt_pet/assets/sprites/thinking/frame_05.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_00.png b/bolt_pet/assets/sprites/walk/frame_00.png new file mode 100644 index 0000000..253de5d Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_00.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_01.png b/bolt_pet/assets/sprites/walk/frame_01.png new file mode 100644 index 0000000..dc391d8 Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_01.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_02.png b/bolt_pet/assets/sprites/walk/frame_02.png new file mode 100644 index 0000000..4b98eae Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_02.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_03.png b/bolt_pet/assets/sprites/walk/frame_03.png new file mode 100644 index 0000000..d7ebe50 Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_03.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_04.png b/bolt_pet/assets/sprites/walk/frame_04.png new file mode 100644 index 0000000..e5704fc Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_04.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_05.png b/bolt_pet/assets/sprites/walk/frame_05.png new file mode 100644 index 0000000..bbbc2aa Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_05.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_06.png b/bolt_pet/assets/sprites/walk/frame_06.png new file mode 100644 index 0000000..b5d4495 Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_06.png differ diff --git a/bolt_pet/assets/sprites/walk/frame_07.png b/bolt_pet/assets/sprites/walk/frame_07.png new file mode 100644 index 0000000..737ef92 Binary files /dev/null and b/bolt_pet/assets/sprites/walk/frame_07.png differ diff --git a/bolt_pet/config.py b/bolt_pet/config.py index 5aaa026..7bed3b1 100644 --- a/bolt_pet/config.py +++ b/bolt_pet/config.py @@ -144,6 +144,30 @@ TTS_STREAMING = os.environ.get("TTS_STREAMING", "true").lower() in ("1", "true", SCREEN_CONTEXT = os.environ.get("SCREEN_CONTEXT", "true").lower() in ("1", "true", "yes", "on") +# ── monitors ──────────────────────────────────────────────────────────────── +# A one-line note about the screen layout (how many, their sizes, which one +# the pet is standing on) rides along with each utterance, so Bolt can decide +# to `petctl jump` somewhere without asking you what you've got plugged in. +# Cheap — the list comes from the UI, nothing is probed per turn. + +MONITOR_CONTEXT = os.environ.get("MONITOR_CONTEXT", "true").lower() in ( + "1", "true", "yes", "on" +) + +# ── screen text (OCR) ─────────────────────────────────────────────────────── +# Lets Bolt actually read a monitor, via `petctl read`. Pull-only: nothing is +# captured unless the server asks for it, and every read is logged. Needs the +# optional capture/OCR extras — see the comments in requirements.txt. +# +# This widens what can leave the machine more than any other switch here: the +# recognised text of a whole screen goes to the server. It is *not* a new +# capability (the shell relay could already run a screenshot tool and OCR it), +# but it is a much easier one to use by accident. Set SCREEN_TEXT=false to +# take it away entirely. + +SCREEN_TEXT = os.environ.get("SCREEN_TEXT", "true").lower() in ("1", "true", "yes", "on") +SCREEN_TEXT_MAX_CHARS = int(os.environ.get("SCREEN_TEXT_MAX_CHARS", "4000")) + # ── quiet hours / do-not-disturb ──────────────────────────────────────────── # Comma-separated HH:MM-HH:MM ranges (wrapping midnight is fine). While # napping the pet dims, stops wandering, and makes no proactive noise — diff --git a/bolt_pet/controller.py b/bolt_pet/controller.py index 2927ff5..9612902 100644 --- a/bolt_pet/controller.py +++ b/bolt_pet/controller.py @@ -20,8 +20,9 @@ from typing import Optional from PySide6.QtCore import QObject, Signal from . import ( - config, history as history_mod, notifications, pet_actions, quiet, - screen_context, server_client, speech_text, updater, + config, history as history_mod, monitors as monitors_mod, notifications, + pet_actions, quiet, screen_context, screen_text, server_client, + speech_text, updater, ) from .audio import barge_in, mic, stt, tts, wake_word from .state import PetState, PetStateMachine @@ -53,6 +54,13 @@ class PetController(QObject): # window. Append-only from this thread; the UI only ever snapshots it. self.history = history_mod.ConversationHistory(limit=config.HISTORY_LIMIT) + # The screen layout, as published by the UI (see set_monitors). Held + # here rather than probed, so "monitor 2" means the same thing to the + # controller and to the window that has to jump there — see + # monitors.py for why that matters. + self._monitors: list[monitors_mod.Monitor] = [] + self._pet_monitor: Optional[int] = None + # Wake-word sensitivity is live-tunable (tray tuner), so it's read # through a callable on every frame rather than captured per listen. self._wake_threshold = config.WAKE_WORD_THRESHOLD @@ -243,7 +251,7 @@ class PetController(QObject): # What's focused right now rides along, so "what's this error?" # has a referent without you having to describe the window. reply = server_client.converse( - screen_context.context_for(text), on_command=self._handle_command + self._with_context(text), on_command=self._handle_command ) except server_client.ServerError as exc: self.log.emit(f"Server error: {exc}") @@ -254,6 +262,31 @@ class PetController(QObject): self._speak(reply) self._state.transition(PetState.IDLE) + def _with_context(self, text: str) -> str: + """Everything the server gets alongside what you actually said: the + focused window title, and a one-line note about the screen layout so + Bolt knows how many monitors there are and where he's standing + without having to ask. Only the *layout* rides along for free — the + text on those screens costs an OCR pass, so it stays behind + `petctl read`.""" + text = screen_context.context_for(text) + if config.MONITOR_CONTEXT: + text = monitors_mod.annotate(text, self._monitors, self._pet_monitor) + return text + + # ── screen layout, published by the UI ─────────────────────────────── + + def set_monitors(self, monitors: list) -> None: + """Slot: the window telling us what screens exist (queued signal).""" + self._monitors = list(monitors) + self.log.emit( + "Screens: " + (monitors_mod.summary(self._monitors) or "none reported") + ) + + def set_pet_monitor(self, index: int) -> None: + """Slot: the window telling us which screen the pet is standing on.""" + self._pet_monitor = int(index) + def _handle_command(self, command: str) -> str: """Server-relayed command. `petctl ...` drives the pet's body and never reaches a shell; everything else is a real command, exactly as @@ -266,11 +299,56 @@ class PetController(QObject): if action is None: return server_client.run_local_command(command) self.log.emit(f"Pet action: {action}") - if action["action"] == "nap": + + # Queries answer from here rather than from pet_actions.describe(): + # their output *is* the useful part, and it's what the server reads + # back off the tool-result relay. + kind = action["action"] + if kind == "monitors": + return monitors_mod.describe(self._monitors, self._pet_monitor) + if kind == "read": + return self._read_screen(action["target"]) + if kind == "jump": + try: + target = monitors_mod.resolve( + self._monitors, action["target"], self._pet_monitor + ) + except ValueError as exc: + self.log.emit(f"petctl jump: {exc}") + return f"[pet] {exc}" + # Hand the window a resolved index, so it can't re-resolve the + # spec against a different screen ordering. + self.action.emit({"action": "jump", "monitor": target.index}) + return f"[pet] jumped to monitor {target.label}" + + if kind == "nap": self.set_napping(bool(action["enabled"])) self.action.emit(action) return pet_actions.describe(action) + def _read_screen(self, target: str) -> str: + """`petctl read` — OCR a screen and hand the text back to the server.""" + if not config.SCREEN_TEXT: + return "[pet] screen reading is disabled (set SCREEN_TEXT=true in .env)" + if not self._monitors: + return "[pet] no monitor information available" + limit = config.SCREEN_TEXT_MAX_CHARS + if target in ("all", "everything", "*"): + self.log.emit(f"Reading all {len(self._monitors)} screens…") + return screen_text.read_monitors(self._monitors, limit) + if target in ("here", "", "this", "current"): + index = self._pet_monitor if self._pet_monitor is not None else 0 + monitor = self._monitors[min(index, len(self._monitors) - 1)] + else: + try: + monitor = monitors_mod.resolve( + self._monitors, target, self._pet_monitor + ) + except ValueError as exc: + return f"[pet] {exc}" + self.log.emit(f"Reading monitor {monitor.number} ({monitor.name})…") + return screen_text.read_monitor(monitor, limit) + def _speak(self, text: str) -> None: self._state.transition(PetState.TALKING) # Bubble gets the markdown stripped but emoji kept (it can't render diff --git a/bolt_pet/monitors.py b/bolt_pet/monitors.py new file mode 100644 index 0000000..9f3990c --- /dev/null +++ b/bolt_pet/monitors.py @@ -0,0 +1,213 @@ +"""Which screens exist, and which one the pet is standing on. + +Deliberately free of Qt *and* of any subprocess probing: the monitor list is +published by the UI (`ui/pet_window.py` builds it from +`QGuiApplication.screens()`) and handed to the controller over a queued +signal, the same way every other UI↔controller message travels. + +That indirection is the whole point. The pet has to agree with itself about +what "monitor 2" means — if the controller enumerated screens with `xrandr` +while the window jumped using Qt's screen list, the two orderings could +disagree and Bolt would announce one screen and land on another. Making Qt the +single source of truth removes that class of bug, and leaves everything here +pure enough to unit test without a display. + +Indices are **1-based in every string a human or the model ever sees**, and +0-based in the list itself. `Monitor.index` is the 0-based one; `.number` is +what gets printed. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable, Optional + +# Directional specs understood by resolve(), mapped to a (dx, dy) heading. +_DIRECTIONS = { + "left": (-1, 0), + "right": (1, 0), + "up": (0, -1), + "above": (0, -1), + "down": (0, 1), + "below": (0, 1), +} + + +@dataclass(frozen=True) +class Monitor: + """One screen, in the global desktop coordinate space.""" + + index: int # 0-based position in the published list + name: str + x: int + y: int + width: int + height: int + primary: bool = False + + @property + def number(self) -> int: + """1-based, for anything a person or the model reads.""" + return self.index + 1 + + @property + def right(self) -> int: + return self.x + self.width + + @property + def bottom(self) -> int: + return self.y + self.height + + @property + def center(self) -> tuple[int, int]: + return self.x + self.width // 2, self.y + self.height // 2 + + def contains(self, x: int, y: int) -> bool: + return self.x <= x < self.right and self.y <= y < self.bottom + + @property + def label(self) -> str: + bits = f"{self.number}: {self.name} {self.width}x{self.height}" + return bits + " (primary)" if self.primary else bits + + +def monitor_containing( + monitors: Iterable[Monitor], x: int, y: int +) -> Optional[Monitor]: + """The screen holding point (x, y), or None if it's off every screen.""" + for monitor in monitors: + if monitor.contains(x, y): + return monitor + return None + + +def nearest_monitor(monitors: Iterable[Monitor], x: int, y: int) -> Optional[Monitor]: + """Screen whose centre is closest to (x, y) — the fallback when a point + lands in the dead space between mismatched screens.""" + monitors = list(monitors) + if not monitors: + return None + return min( + monitors, + key=lambda m: (m.center[0] - x) ** 2 + (m.center[1] - y) ** 2, + ) + + +def resolve( + monitors: list[Monitor], spec: str, current: Optional[int] = None +) -> Monitor: + """Turn a `petctl jump` target into a screen. + + Accepts a 1-based number, a name (case-insensitive substring, so "hdmi" + finds "HDMI-0"), `next`/`prev`, `primary`, `other`, or a direction + (`left`/`right`/`up`/`down`) relative to *current*. Raises ValueError with + a message meant to be read by the model, since it goes back as tool + output. + """ + if not monitors: + raise ValueError("no monitors have been reported yet") + spec = (spec or "").strip().lower() + if not spec: + raise ValueError("jump needs a target monitor") + count = len(monitors) + if current is None or not (0 <= current < count): + current = next((m.index for m in monitors if m.primary), 0) + + if spec.isdigit(): + number = int(spec) + if not (1 <= number <= count): + raise ValueError( + f"there is no monitor {number}; you have {count} " + f"(1-{count})" + ) + return monitors[number - 1] + + if spec in ("next", "forward"): + return monitors[(current + 1) % count] + if spec in ("prev", "previous", "back"): + return monitors[(current - 1) % count] + if spec == "primary": + return next((m for m in monitors if m.primary), monitors[0]) + if spec == "other": + # With two screens "the other one" is unambiguous; with more it's just + # the next one round, which is at least always a *different* screen. + return monitors[(current + 1) % count] + if spec == "random": + # Deterministic-free choice is the caller's business; pick the screen + # furthest from the current one so "random" always visibly moves. + here = monitors[current].center + return max( + monitors, + key=lambda m: (m.center[0] - here[0]) ** 2 + (m.center[1] - here[1]) ** 2, + ) + + if spec in _DIRECTIONS: + dx, dy = _DIRECTIONS[spec] + here = monitors[current].center + candidates = [] + for monitor in monitors: + if monitor.index == current: + continue + ox, oy = monitor.center + along = (ox - here[0]) * dx + (oy - here[1]) * dy + if along <= 0: + continue # not in that direction at all + drift = abs((ox - here[0]) * dy + (oy - here[1]) * dx) + candidates.append((drift, along, monitor)) + if not candidates: + raise ValueError( + f"there's no monitor to the {spec} of monitor " + f"{monitors[current].number}" + ) + # Prefer the best-aligned screen, then the closest of those. + candidates.sort(key=lambda item: (item[0], item[1])) + return candidates[0][2] + + matches = [m for m in monitors if spec in m.name.lower()] + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + raise ValueError( + f"{spec!r} matches several monitors: " + + ", ".join(m.label for m in matches) + ) + raise ValueError( + f"unknown monitor {spec!r}; you have: " + + "; ".join(m.label for m in monitors) + + " — or use next/prev/primary/left/right/up/down" + ) + + +def describe(monitors: list[Monitor], current: Optional[int] = None) -> str: + """Full listing, used as `petctl monitors` output.""" + if not monitors: + return "[pet] no monitor information available" + lines = [f"[pet] {len(monitors)} monitor(s):"] + for monitor in monitors: + here = " <- Bolt is here" if monitor.index == current else "" + lines.append( + f" {monitor.label} at +{monitor.x}+{monitor.y}{here}" + ) + return "\n".join(lines) + + +def summary(monitors: list[Monitor], current: Optional[int] = None) -> Optional[str]: + """One-line version tacked onto each utterance — short on purpose, since + it rides along with every single thing you say.""" + if not monitors: + return None + parts = ", ".join(f"{m.number}) {m.name} {m.width}x{m.height}" for m in monitors) + line = f"{len(monitors)} monitors: {parts}" + if current is not None and 0 <= current < len(monitors): + line += f"; Bolt is on {monitors[current].number}" + return line + + +def annotate(text: str, monitors: list[Monitor], current: Optional[int] = None) -> str: + """Attach the screen summary as a separate aside, matching the style of + screen_context.annotate() so the model can ignore it when irrelevant.""" + text = (text or "").strip() + line = summary(monitors, current) + if not text or not line: + return text + return f"{text}\n\n[{line}]" diff --git a/bolt_pet/pet_actions.py b/bolt_pet/pet_actions.py index 5e31da4..c9376b9 100644 --- a/bolt_pet/pet_actions.py +++ b/bolt_pet/pet_actions.py @@ -29,8 +29,18 @@ ANCHORS = ( EMOTES = ("wave", "hop", "spin", "nod", "shake", "bounce", "wiggle") +# Where `petctl jump` can be aimed. A bare number (1-based) works too, as does +# any unique part of a monitor's name — resolution lives in monitors.resolve(). +MONITOR_SPECS = ( + "next", "prev", "primary", "other", "random", + "left", "right", "up", "down", +) + HELP = ( "petctl move | <" + "|".join(ANCHORS) + ">\n" + "petctl jump \n" + "petctl monitors\n" + "petctl read [monitor number|here|all]\n" "petctl emote <" + "|".join(EMOTES) + ">\n" "petctl say \n" "petctl wander on|off\n" @@ -82,6 +92,24 @@ def parse(command: str) -> Optional[dict]: raise ActionError(f"unknown position {args[0]!r}; try one of: " + ", ".join(ANCHORS)) return {"action": "move", "anchor": anchor} + if verb in ("jump", "monitor", "screen"): + if not args: + raise ActionError( + "jump needs a monitor: a number, a name, or one of " + + ", ".join(MONITOR_SPECS) + ) + # The spec isn't validated here on purpose: which monitors exist is a + # runtime fact this pure module doesn't have. monitors.resolve() does + # it once the published screen list is in hand. + return {"action": "jump", "target": " ".join(args).strip()} + + if verb in ("monitors", "screens", "displays"): + return {"action": "monitors"} + + if verb in ("read", "look", "ocr", "see"): + target = (" ".join(args).strip() or "here").lower() + return {"action": "read", "target": target} + if verb in ("emote", "do"): if not args: raise ActionError("emote needs a name: " + ", ".join(EMOTES)) @@ -124,6 +152,8 @@ def describe(action: dict) -> str: if kind == "move": where = action.get("anchor") or f"({action.get('x')}, {action.get('y')})" return f"[pet] walking to {where}" + if kind == "jump": + return f"[pet] jumping to monitor {action['target']}" if kind == "emote": return f"[pet] {action['emote']}" if kind == "say": diff --git a/bolt_pet/screen_text.py b/bolt_pet/screen_text.py new file mode 100644 index 0000000..a462e5d --- /dev/null +++ b/bolt_pet/screen_text.py @@ -0,0 +1,187 @@ +"""Reading the text that's actually on a monitor, via screenshot + OCR. + +This is the "Bolt can see what's on screen" half of the screen features. It is +**pull, not push**: nothing here runs on its own. The server has to ask, by +relaying `petctl read`, and the recognised text goes back as that command's +output through the existing tool-result relay (see server_client.converse). +That's deliberate on two counts — OCR of a 4K screen costs a second or two, +which would be tacked onto every single utterance if it ran automatically, and +"screen contents leave this machine" should be a thing Bolt decides to do and +you can see in the log, not a silent constant. + +Both halves are optional and soft-fail with a reason, the way hotkey.py does: +capture needs `mss`, recognition needs a Tesseract or RapidOCR install. With +neither, `petctl read` reports what's missing instead of raising, and the rest +of the pet carries on. + +The pure parts (cleaning OCR output, formatting the reply, deciding which +engine to use given what's installed) are split out and unit tested; only +capture and the OCR call itself need a real screen. +""" + +from __future__ import annotations + +import re +import shutil +from typing import Callable, Optional + +from .monitors import Monitor + +DEFAULT_MAX_CHARS = 4000 + +INSTALL_HINT = ( + "install one of: `pip install mss pytesseract` + `sudo apt install " + "tesseract-ocr` (fastest), or `pip install mss rapidocr-onnxruntime` " + "(no system package needed)" +) + +# Lines that are almost certainly OCR noise rather than text: window chrome +# fragments, isolated punctuation, single stray characters. +_MIN_MEANINGFUL = 2 + + +def _module_available(name: str) -> bool: + import importlib.util + + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + +# ── pure helpers (unit tested; no screen, no OCR engine needed) ────────────── + +def resolve_engine( + has_module: Callable[[str], bool] = _module_available, + which: Callable[[str], Optional[str]] = shutil.which, +) -> tuple[Optional[str], str]: + """Pick an OCR engine from what's installed. + + Returns `(engine, reason)`. *engine* is None when nothing usable is + present, and *reason* then explains what to install. Probes are injected + so this is testable on a machine with a different set of things installed. + """ + if has_module("pytesseract") and which("tesseract"): + return "pytesseract", "" + if has_module("rapidocr_onnxruntime"): + return "rapidocr", "" + if has_module("pytesseract") and not which("tesseract"): + return None, ( + "pytesseract is installed but the tesseract binary isn't on PATH " + "(try: sudo apt install tesseract-ocr)" + ) + return None, f"no OCR engine available — {INSTALL_HINT}" + + +def capture_available(has_module: Callable[[str], bool] = _module_available) -> bool: + return has_module("mss") + + +def clean_ocr_text(raw: str, max_chars: int = DEFAULT_MAX_CHARS) -> str: + """Squeeze raw OCR output into something worth sending. + + Screen OCR produces a lot of junk — single stray glyphs off window + borders, runs of blank lines, the same toolbar label recognised twice. All + of that costs tokens and tells the model nothing, so it goes. + """ + if not raw: + return "" + lines: list[str] = [] + for line in raw.splitlines(): + line = re.sub(r"[^\S\n]+", " ", line).strip() + if not line: + continue + if len(re.sub(r"[^0-9A-Za-z]", "", line)) < _MIN_MEANINGFUL: + continue + if lines and line == lines[-1]: + continue # consecutive duplicate + lines.append(line) + text = "\n".join(lines) + if max_chars and len(text) > max_chars: + text = text[: max_chars - 1].rstrip() + "…" + text += "\n[truncated]" + return text + + +def format_reading(monitor: Optional[Monitor], text: str) -> str: + """The tool output handed back for `petctl read`.""" + where = f"monitor {monitor.number} ({monitor.name})" if monitor else "screen" + if not text.strip(): + return f"[pet] read {where}: no text recognised" + return f"[pet] text on {where}:\n{text}" + + +# ── capture + recognition (needs a real screen) ────────────────────────────── + +def capture(monitor: Monitor): + """Grab *monitor* as a PIL image, or None if capture isn't available.""" + try: + import mss + from PIL import Image + except ImportError: + return None + try: + box = { + "left": monitor.x, + "top": monitor.y, + "width": monitor.width, + "height": monitor.height, + } + with mss.mss() as sct: + shot = sct.grab(box) + return Image.frombytes("RGB", shot.size, shot.bgra, "raw", "BGRX") + except Exception: + return None + + +def _ocr(image, engine: str) -> str: + if engine == "pytesseract": + import pytesseract + + # Grayscale first: tesseract is measurably better on it than on the + # colour desktop, and it's a cheap conversion. + return pytesseract.image_to_string(image.convert("L")) + if engine == "rapidocr": + import numpy as np + from rapidocr_onnxruntime import RapidOCR + + result, _ = RapidOCR()(np.array(image)) + if not result: + return "" + return "\n".join(line[1] for line in result) + return "" + + +def read_monitor(monitor: Monitor, max_chars: int = DEFAULT_MAX_CHARS) -> str: + """OCR one screen and return the formatted tool output. + + Never raises: every failure path returns a sentence explaining itself, + because the return value goes straight back to the server as the result of + a command Bolt chose to run. + """ + if not capture_available(): + return f"[pet] can't capture the screen — {INSTALL_HINT}" + engine, reason = resolve_engine() + if engine is None: + return f"[pet] can't read the screen — {reason}" + image = capture(monitor) + if image is None: + return ( + f"[pet] couldn't capture monitor {monitor.number} " + "(is this a Wayland session? mss needs X11)" + ) + try: + raw = _ocr(image, engine) + except Exception as exc: + return f"[pet] OCR failed on monitor {monitor.number}: {exc}" + return format_reading(monitor, clean_ocr_text(raw, max_chars)) + + +def read_monitors(monitors: list[Monitor], max_chars: int = DEFAULT_MAX_CHARS) -> str: + """OCR several screens, splitting the character budget between them.""" + if not monitors: + return "[pet] no monitor information available" + if len(monitors) == 1: + return read_monitor(monitors[0], max_chars) + share = max(400, max_chars // len(monitors)) + return "\n\n".join(read_monitor(m, share) for m in monitors) diff --git a/bolt_pet/ui/app.py b/bolt_pet/ui/app.py index 3bf96e3..694cb71 100644 --- a/bolt_pet/ui/app.py +++ b/bolt_pet/ui/app.py @@ -46,6 +46,13 @@ def run() -> int: controller.finished.connect(thread.quit) window.talk_requested.connect(controller.request_talk_now) + # The window owns the screen list and tells the controller about it, so + # both ends agree on what "monitor 2" means (see monitors.py). + window.monitors_changed.connect(controller.set_monitors) + window.pet_monitor_changed.connect(controller.set_pet_monitor) + # PetWindow publishes once in its constructor, which ran before those + # connections existed — so say it again now that anyone is listening. + window.publish_monitors() window.copied.connect(lambda text: _log(f"Copied to clipboard: {text[:60]}")) history_window = HistoryWindow(controller.history) diff --git a/bolt_pet/ui/pet_window.py b/bolt_pet/ui/pet_window.py index 830e8c6..075813e 100644 --- a/bolt_pet/ui/pet_window.py +++ b/bolt_pet/ui/pet_window.py @@ -20,8 +20,9 @@ from PySide6.QtGui import ( from PySide6.QtWidgets import QApplication, QWidget from .. import config +from ..monitors import Monitor from ..state import PetState -from .sprite import SpriteSet +from .sprite import WALK, SpriteSet _DRAG_THRESHOLD_PX = 4 # Movement runs on its own ~30fps timer, independent of the (slower) sprite @@ -29,6 +30,12 @@ _DRAG_THRESHOLD_PX = 4 _WANDER_TICK_MS = 33 _EMOTE_TICKS = 36 # ~1.2s per emote at the tick rate above _NAP_OPACITY = 0.35 +# How far the pet travels per walk-cycle frame. The cycle is advanced by +# distance rather than by the animation clock so a planted paw tracks backwards +# at exactly the speed the window moves forwards — drive it off a timer instead +# and the feet skate whenever PET_WANDER_SPEED doesn't happen to match the fps. +# Eight frames at 13px is a ~104px stride cycle, a bit under the pet's width. +_WALK_PIXELS_PER_FRAME = 13.0 def emote_transform(emote: str, progress: float) -> tuple[float, float, float, float]: @@ -164,6 +171,12 @@ class SpeechBubble(QWidget): class PetWindow(QWidget): talk_requested = Signal() copied = Signal(str) # bubble text the user just put on the clipboard + # The screen layout, published *to* the controller (queued, cross-thread). + # The window is the only thing allowed to ask Qt about screens, so the + # controller and the window can never disagree about what "monitor 2" + # means — see monitors.py. + monitors_changed = Signal(list) # list[monitors.Monitor] + pet_monitor_changed = Signal(int) # 0-based index the pet is standing on def __init__(self, sprite_dir: Optional[Path] = None, size: Optional[int] = None): super().__init__() @@ -201,6 +214,10 @@ class PetWindow(QWidget): self._next_wander_at = 0.0 self._bob_offset = 0 self._bob_phase = 0.0 + self._walking = False + self._facing = 1 # +1 right, -1 left; the walk art is drawn facing right + self._walk_distance = 0.0 + self._mirror_cache: dict[int, QPixmap] = {} self._schedule_next_wander() self._wander_timer = QTimer(self) self._wander_timer.timeout.connect(self._movement_tick) @@ -210,6 +227,18 @@ class PetWindow(QWidget): self.set_click_through(config.PET_CLICK_THROUGH) self._place_start_position() + self._monitors: list[Monitor] = [] + self._pet_monitor: Optional[int] = None + self._last_published_pos: Optional[QPoint] = None + app = QApplication.instance() + if app is not None: + # Screens come and go — a laptop docking, a TV waking up. Republish + # rather than letting Bolt jump to a monitor that's been unplugged. + app.screenAdded.connect(lambda _s: self.publish_monitors()) + app.screenRemoved.connect(lambda _s: self.publish_monitors()) + app.primaryScreenChanged.connect(lambda _s: self.publish_monitors()) + self.publish_monitors() + # ── placement ──────────────────────────────────────────────────────── def _place_start_position(self) -> None: @@ -244,6 +273,8 @@ class PetWindow(QWidget): if target is not None: self._wander_target = target self._commanded_move = True # overrides the idle-only rule + elif kind == "jump": + self.jump_to_monitor(int(action["monitor"])) elif kind == "emote": self.start_emote(action["emote"]) elif kind == "say": @@ -378,6 +409,98 @@ class PetWindow(QWidget): """Stroll immediately (tray menu / anything that wants a nudge).""" self._next_wander_at = 0.0 + # ── monitors ───────────────────────────────────────────────────────── + + def _build_monitors(self) -> list[Monitor]: + """Snapshot Qt's screen list as plain dataclasses. + + Full `geometry()`, not `availableGeometry()`: these coordinates are + what a screen grab gets cropped to, and a grab doesn't stop at the + taskbar. Placement uses availableGeometry separately. + """ + primary = QApplication.primaryScreen() + out = [] + for index, screen in enumerate(QApplication.screens()): + geo = screen.geometry() + out.append( + Monitor( + index=index, + name=screen.name() or f"screen-{index + 1}", + x=geo.x(), + y=geo.y(), + width=geo.width(), + height=geo.height(), + primary=screen is primary, + ) + ) + return out + + def publish_monitors(self, force: bool = True) -> None: + """Push the current layout to whoever's listening (the controller). + + *force* re-emits even when nothing changed, which is what the initial + wiring in ui/app.py needs: this window is built before the controller + exists, so the constructor's first publish goes to nobody. + """ + monitors = self._build_monitors() + changed = monitors != self._monitors + self._monitors = monitors + if changed or force: + self.monitors_changed.emit(monitors) + self._publish_pet_monitor(force=True) + + def monitors(self) -> list[Monitor]: + return list(self._monitors) + + def current_monitor_index(self) -> Optional[int]: + center = self.frameGeometry().center() + screens = QApplication.screens() + if not screens: + return None + screen = QApplication.screenAt(center) + if screen is not None: + try: + return screens.index(screen) + except ValueError: + pass + # Straddling a gap or dragged off the desktop entirely — fall back to + # whichever screen centre is nearest rather than reporting nothing. + best = min( + range(len(screens)), + key=lambda i: (screens[i].geometry().center() - center).manhattanLength(), + ) + return best + + def _publish_pet_monitor(self, force: bool = False) -> None: + index = self.current_monitor_index() + if index is None: + return + if force or index != self._pet_monitor: + self._pet_monitor = index + self.pet_monitor_changed.emit(index) + + def jump_to_monitor(self, index: int) -> None: + """Teleport to *index* (0-based, resolved by the controller) and land + with a hop. Instant rather than a stroll — Bolt asked to *jump*, and + walking between screens would take the long way across the desktop.""" + screens = QApplication.screens() + if not (0 <= index < len(screens)): + return + geo = screens[index].availableGeometry() + point = self._clamp_to_screen( + QPoint( + geo.left() + (geo.width() - self.width()) // 2, + geo.top() + (geo.height() - self.height()) // 2, + ), + geo, + ) + self._stop_walking() # drop any stroll in flight, or it walks straight back + self.move(point) + self._schedule_next_wander() + self._publish_pet_monitor(force=True) + self.start_emote("hop") + self.update() + def _screen_geometry(self): # screenAt() so a multi-monitor setup keeps the pet on the screen # it's currently standing on rather than yanking it to the primary. @@ -389,12 +512,17 @@ class PetWindow(QWidget): self._next_wander_at = time.monotonic() + random.uniform(0.5 * base, 1.5 * base) def _stop_walking(self) -> None: - if self._wander_target is None and not self._bob_offset: + if self._wander_target is None and not self._bob_offset and not self._walking: return self._wander_target = None self._commanded_move = False self._bob_phase = 0.0 self._bob_offset = 0 + self._walking = False + self._walk_distance = 0.0 + # Back to a standing frame, so the next stroll starts from a contact + # pose instead of mid-stride. + self.sprites.get(WALK).reset() self.update() def snap_to_edge(self) -> bool: @@ -447,6 +575,13 @@ class PetWindow(QWidget): wandering only happens when it's otherwise unoccupied.""" self._advance_emote() self._wander_tick() + # Report crossing a screen boundary — by strolling, by being dragged, + # by anything. Guarded on the position actually changing so the common + # case (a stationary pet, 30x a second) costs one comparison. + position = self.pos() + if position != self._last_published_pos: + self._last_published_pos = position + self._publish_pet_monitor() def _wander_tick(self) -> None: # Only stroll while genuinely idle: not mid-drag, not napping, not @@ -484,11 +619,51 @@ class PetWindow(QWidget): self.snap_to_edge() else: self.move(round(here.x() + dx / distance * step), round(here.y() + dy / distance * step)) - self._bob_phase += 0.45 # little walk-cycle hop - self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase)))) + self._advance_walk(dx, dy, step) self.update() self._reposition_bubble() + def _advance_walk(self, dx: float, dy: float, step: float) -> None: + """Drive the walk cycle from distance travelled (see the constant). + + Falls back to the old bob-in-code if there's no walk art, so a sprite + folder without a walk/ directory still looks like it's moving rather + than sliding perfectly flat. + """ + self._walking = True + # Only turn on meaningful horizontal travel: a near-vertical stroll + # would otherwise flip him back and forth on rounding noise. + if abs(dx) > 1.0: + self._facing = 1 if dx > 0 else -1 + if not self.sprites.has(WALK): + self._bob_phase += 0.45 + self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase)))) + return + self._bob_offset = 0 # the walk frames carry their own weight shift + self._walk_distance += step + while self._walk_distance >= _WALK_PIXELS_PER_FRAME: + self._walk_distance -= _WALK_PIXELS_PER_FRAME + self.sprites.get(WALK).advance() + + def _animation_key(self): + """Walking overrides the state animation — but only while genuinely + idle-and-moving, so he doesn't trot on the spot mid-sentence.""" + if self._walking and self.sprites.has(WALK): + return WALK + return self._current_state + + def _oriented(self, pixmap: Optional[QPixmap]) -> Optional[QPixmap]: + """Mirror the (right-facing) walk art when he's heading left. Cached + per source frame — flipping on every paint would be wasteful at 30fps.""" + if pixmap is None or self._facing >= 0: + return pixmap + key = pixmap.cacheKey() + mirrored = self._mirror_cache.get(key) + if mirrored is None: + mirrored = pixmap.transformed(QTransform().scale(-1, 1), Qt.SmoothTransformation) + self._mirror_cache[key] = mirrored + return mirrored + # ── state / speech ────────────────────────────────────────────────── def set_state(self, state: PetState) -> None: @@ -514,6 +689,11 @@ class PetWindow(QWidget): # ── animation ──────────────────────────────────────────────────────── def _advance_frame(self) -> None: + # While walking the cycle is stepped by _advance_walk from distance + # travelled; letting this timer also advance it would double-step it + # and put the feet out of sync with the movement. + if self._walking and self.sprites.has(WALK): + return self.sprites.get(self._current_state).advance() self.update() @@ -521,7 +701,10 @@ class PetWindow(QWidget): painter = QPainter(self) painter.setRenderHint(QPainter.Antialiasing) painter.setRenderHint(QPainter.SmoothPixmapTransform) - pixmap: Optional[QPixmap] = self.sprites.get(self._current_state).current() + key = self._animation_key() + pixmap: Optional[QPixmap] = self.sprites.get(key).current() + if key == WALK: + pixmap = self._oriented(pixmap) if pixmap is None: self._apply_input_mask(None, 0, 0) return diff --git a/bolt_pet/ui/sprite.py b/bolt_pet/ui/sprite.py index 7e2fc12..33060b6 100644 --- a/bolt_pet/ui/sprite.py +++ b/bolt_pet/ui/sprite.py @@ -95,17 +95,46 @@ def _load_frames_from_dir(directory: Path, size: int) -> list[QPixmap]: return frames +WALK = "walk" + +# Animations that aren't pipeline states. Walking is a property of *movement*, +# orthogonal to whether the pet is idle/listening/talking, so it deliberately +# isn't a PetState — state.py stays a description of the conversation, not of +# the body. Loaded the same way, keyed by name. +EXTRA_ANIMATIONS = (WALK,) + + class SpriteSet: - """All animations for every PetState, loaded from *sprite_dir*.""" + """All animations for every PetState, plus the extras, from *sprite_dir*.""" def __init__(self, sprite_dir: Path = DEFAULT_SPRITE_DIR, size: int = 160): self.size = size - self._animations: dict[PetState, SpriteAnimation] = {} + self._animations: dict[str, SpriteAnimation] = {} + self._loaded: set[str] = set() # keys backed by real art, not placeholders for state in PetState: frames = _load_frames_from_dir(sprite_dir / state.value, size) - if not frames: + if frames: + self._loaded.add(state.value) + else: frames = _placeholder_frames(state, size) - self._animations[state] = SpriteAnimation(frames) + self._animations[state.value] = SpriteAnimation(frames) + for name in EXTRA_ANIMATIONS: + frames = _load_frames_from_dir(sprite_dir / name, size) + if frames: + self._loaded.add(name) + self._animations[name] = SpriteAnimation(frames) - def get(self, state: PetState) -> SpriteAnimation: - return self._animations[state] + @staticmethod + def _key(key) -> str: + return key.value if isinstance(key, PetState) else str(key) + + def get(self, key) -> SpriteAnimation: + """Animation for a PetState or an extra name. Unknown/absent extras + fall back to idle, so a sprite folder with no walk/ still runs.""" + return self._animations.get(self._key(key)) or self._animations[PetState.IDLE.value] + + def has(self, key) -> bool: + """True only when real frames were found — the caller uses this to + decide whether to use an extra animation at all, rather than being + handed a placeholder blob that looks nothing like walking.""" + return self._key(key) in self._loaded diff --git a/requirements.txt b/requirements.txt index 7c7cc56..54b6124 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,18 @@ pynput>=1.7 # Not imported by the app itself. Pillow>=10.0 +# Screen reading (`petctl read`) — Bolt OCRs a monitor and uses the text in +# his reply. Both optional: without them `petctl read` reports what's missing +# and the rest of the pet is unaffected. +# mss screen capture. X11/Win32/macOS — NOT Wayland. +# pytesseract a thin wrapper; the actual engine is a system package: +# sudo apt install tesseract-ocr +# No-sudo alternative to those two lines: pip install rapidocr-onnxruntime +# (pure pip, reuses the onnxruntime openwakeword already pulls in, slower to +# start). screen_text.resolve_engine() picks whichever is present. +mss>=9.0 +pytesseract>=0.3.10 + # Test runner (tests/ — pure logic, no audio hardware or display needed; # run with QT_QPA_PLATFORM=offscreen). pytest>=8.0 diff --git a/scripts/generate_bolt_sprites.py b/scripts/generate_bolt_sprites.py new file mode 100644 index 0000000..ea0f186 --- /dev/null +++ b/scripts/generate_bolt_sprites.py @@ -0,0 +1,779 @@ +"""Draw Bolt — the pet — as per-state PNG frame sequences. + +Produces the `assets/sprites//frame_NN.png` convention that +`bolt_pet/ui/sprite.py` loads (see `assets/sprites/README.md`). The art is +generated rather than sourced so it stays editable: tweak a colour or a pose +parameter here and re-run, instead of hand-editing 24 PNGs. + + python scripts/generate_bolt_sprites.py # write into the real asset dir + python scripts/generate_bolt_sprites.py --out /tmp/prev # preview somewhere else + +Everything is drawn in normalised 0..1 coordinates on a square canvas and +super-sampled `SS`x before being downscaled, because PIL's draw primitives +have no antialiasing of their own. +""" + +from __future__ import annotations + +import argparse +import math +from pathlib import Path + +from PIL import Image, ImageDraw + +SS = 4 # supersampling factor +OUT = 320 # final frame size (2x the default PET_SIZE of 160) +S = OUT * SS + +# --- palette --------------------------------------------------------------- +# A cream shepherd-ish pup with a slate cap, amber eyes and a lightning blaze. +C_OUTLINE = (34, 42, 58, 255) +C_FUR = (246, 244, 238, 255) +C_FUR_SHADE = (214, 210, 200, 255) +C_DARK = (78, 92, 122, 255) +C_DARK2 = (58, 70, 96, 255) +C_INNER_EAR = (226, 154, 158, 255) +C_BROW = (206, 166, 118, 255) +# The far side of the walking pose. Distinctly darker than C_FUR_SHADE, which +# is too close to the cream to read as "behind the dog" at 160px. +C_FUR_FAR = (168, 176, 192, 255) +C_NOSE = (40, 48, 66, 255) +C_IRIS = (196, 128, 50, 255) +C_PUPIL = (30, 36, 50, 255) +C_WHITE = (255, 255, 255, 255) +C_BOLT = (255, 206, 61, 255) +C_COLLAR = (222, 84, 46, 255) +C_TAG = (255, 198, 68, 255) +C_TONGUE = (230, 116, 128, 255) +C_GLOW = (92, 214, 244, 255) + +# --- layout constants (normalised) ----------------------------------------- +HEAD_CX, HEAD_CY = 0.50, 0.375 +HEAD_W, HEAD_H = 0.50, 0.44 +NECK_Y = 0.565 # head layer rotates about here so tilts pivot at the neck +EAR_PIVOT = 0.335, 0.275 + +OW = 0.0105 # outline width, normalised + + +def px(v: float) -> float: + return v * S + + +def _w(width: float) -> int: + return max(1, int(round(px(width)))) + + +def ell(d, cx, cy, w, h, fill, outline=C_OUTLINE, ow=OW): + d.ellipse( + [px(cx - w / 2), px(cy - h / 2), px(cx + w / 2), px(cy + h / 2)], + fill=fill, + outline=outline, + width=_w(ow) if outline else 0, + ) + + +def rrect(d, cx, cy, w, h, r, fill, outline=C_OUTLINE, ow=OW): + d.rounded_rectangle( + [px(cx - w / 2), px(cy - h / 2), px(cx + w / 2), px(cy + h / 2)], + radius=px(r), + fill=fill, + outline=outline, + width=_w(ow) if outline else 0, + ) + + +def poly(d, pts, fill, outline=C_OUTLINE, ow=OW): + d.polygon( + [(px(x), px(y)) for x, y in pts], + fill=fill, + outline=outline, + width=_w(ow) if outline else 0, + ) + + +def rotate_pts(pts, pivot, deg): + a = math.radians(deg) + ca, sa = math.cos(a), math.sin(a) + ox, oy = pivot + out = [] + for x, y in pts: + dx, dy = x - ox, y - oy + out.append((ox + dx * ca - dy * sa, oy + dx * sa + dy * ca)) + return out + + +def lerp(a, b, t): + return a + (b - a) * t + + +def bolt_shape(cx, cy, w, h): + """A lightning bolt polygon in a (w x h) box centred on (cx, cy).""" + unit = [ + (0.62, 0.00), + (0.10, 0.56), + (0.44, 0.56), + (0.28, 1.00), + (0.90, 0.40), + (0.55, 0.40), + (0.80, 0.00), + ] + return [(cx + (u - 0.5) * w, cy + (v - 0.5) * h) for u, v in unit] + + +# --- body ------------------------------------------------------------------ +def _tail_points(p, steps=26): + """Quadratic-bezier spine of the tail as (x, y, radius) samples. + + Shared by the fill and outline passes so a wag can't move one and not the + other. The base sits deep inside the haunch, which is drawn over it, so + the tail reads as growing out of the body rather than floating beside it. + """ + wag = p["tail"] + base = (0.620, 0.845) + ctrl = (0.955, 0.870 - 0.025 * wag) + end = (0.905, 0.605 - 0.065 * wag) + pts = [] + for i in range(steps + 1): + t = i / steps + x = (1 - t) ** 2 * base[0] + 2 * (1 - t) * t * ctrl[0] + t**2 * end[0] + y = (1 - t) ** 2 * base[1] + 2 * (1 - t) * t * ctrl[1] + t**2 * end[1] + pts.append((x, y, lerp(0.080, 0.042, t))) + return pts + + +def draw_tapered(d, pts, color_at): + """Draw a tapered limb from (x, y, radius) samples. + + Two passes: circles along the spine for the fill, then the two silhouette + edges, so it reads as one solid shape instead of a string of beads. + *color_at* takes 0..1 along the length, which is how the tail gets its + cream tip. + """ + last = len(pts) - 1 + for i, (x, y, r) in enumerate(pts): + ell(d, x, y, r * 2, r * 2, color_at(i / last), outline=None) + for side in (1, -1): + edge = [] + for i, (x, y, r) in enumerate(pts): + j = min(i + 1, last) + k = max(i - 1, 0) + tx, ty = pts[j][0] - pts[k][0], pts[j][1] - pts[k][1] + n = math.hypot(tx, ty) or 1e-6 + nx, ny = -ty / n, tx / n + edge.append((x + nx * r * side, y + ny * r * side)) + d.line([(px(x), px(y)) for x, y in edge], fill=C_OUTLINE, width=_w(OW), joint="curve") + x, y, r = pts[last] + ell(d, x, y, r * 2, r * 2, color_at(1.0)) + + +def draw_tail(d, p): + # Only the last stretch is the cream tip. The haunch hides the first ~half + # of the tail, so a generous tip leaves the visible part looking like a + # pale blob floating next to the dog rather than its tail. + draw_tapered(d, _tail_points(p), lambda t: C_DARK if t < 0.84 else C_FUR) + + +def draw_body(d, p): + br = p["breathe"] + # haunches (sitting) + ell(d, 0.285, 0.795, 0.235, 0.275, C_DARK) + ell(d, 0.715, 0.795, 0.235, 0.275, C_DARK) + # torso + ell(d, 0.50, 0.745 - 0.004 * br, 0.455 + 0.012 * br, 0.395 + 0.014 * br, C_DARK) + # front legs + for cx in (0.415, 0.585): + rrect(d, cx, 0.845, 0.125, 0.215, 0.062, C_FUR) + ell(d, cx, 0.925, 0.155, 0.095, C_FUR) + # chest / belly blaze + ell(d, 0.50, 0.735 - 0.004 * br, 0.275 + 0.008 * br, 0.315 + 0.012 * br, C_FUR) + # toes + for cx in (0.415, 0.585): + for off in (-0.035, 0.0, 0.035): + d.arc( + [px(cx + off - 0.017), px(0.902), px(cx + off + 0.017), px(0.945)], + start=250, + end=290, + fill=C_FUR_SHADE, + width=_w(0.007), + ) + + +def draw_collar(d, p): + rrect(d, 0.50, 0.585, 0.315, 0.062, 0.031, C_COLLAR) + tag = C_GLOW if p.get("tag_glow") else C_TAG + ell(d, 0.50, 0.638, 0.082, 0.082, tag) + poly(d, bolt_shape(0.50, 0.638, 0.030, 0.052), C_OUTLINE, outline=None) + + +# --- head ------------------------------------------------------------------ +# Ear outline in a *local* frame: origin at the base on the skull, +x points +# outward (away from the muzzle), +y points up. Keeping it side-agnostic here +# and mirroring at draw time avoids sign confusion — an earlier version mixed +# the conventions and the ears flattened into a brim whenever they rotated. +_EAR_LOCAL = [ + (-0.058, -0.038), + (0.078, -0.038), + (0.092, 0.140), + (0.030, 0.248), + (-0.038, 0.122), +] + + +def _ear_polygon(side, lean_deg): + """Mirror + lean the local ear, returning canvas-space points. + + *lean_deg* tips the ear away from vertical: 0 is fully perked, larger + values relax and eventually droop it out sideways. + """ + a = math.radians(lean_deg) + ca, sa = math.cos(a), math.sin(a) + pivot_x = 0.5 + side * (0.5 - EAR_PIVOT[0]) + pts = [] + for x, y in _EAR_LOCAL: + rx = x * ca + y * sa + ry = -x * sa + y * ca + pts.append((pivot_x + side * rx, EAR_PIVOT[1] - ry)) + return pts + + +def draw_ears(d, p): + perk = p["ear"] + twitch = p.get("ear_twitch", 0.0) + for side in (-1, 1): + lean = 18.0 * (1.0 - perk) + 44.0 * max(0.0, -perk) + if side == 1: + lean -= twitch * 12.0 + pts = _ear_polygon(side, lean) + poly(d, pts, C_DARK) + base_mid = ( + (pts[0][0] + pts[1][0]) / 2, + (pts[0][1] + pts[1][1]) / 2, + ) + inner = [(lerp(base_mid[0], x, 0.60), lerp(base_mid[1], y, 0.64)) for x, y in pts] + poly(d, inner, C_INNER_EAR, outline=None) + + +def draw_cap(layer, p): + """Slate cap over the top of the head, clipped to the head silhouette.""" + mask = Image.new("L", (S, S), 0) + ImageDraw.Draw(mask).ellipse( + [ + px(HEAD_CX - HEAD_W / 2), + px(HEAD_CY - HEAD_H / 2), + px(HEAD_CX + HEAD_W / 2), + px(HEAD_CY + HEAD_H / 2), + ], + fill=255, + ) + cap = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + dc = ImageDraw.Draw(cap) + ell(dc, HEAD_CX, 0.245, 0.54, 0.30, C_DARK, outline=None) + # brow dip between the eyes, so the cap reads as a marking not a helmet + ell(dc, HEAD_CX, 0.352, 0.155, 0.115, C_FUR, outline=None) + cap.putalpha(Image.composite(cap.getchannel("A"), Image.new("L", (S, S), 0), mask)) + layer.alpha_composite(cap) + + +def draw_eyes(d, p): + blink = p["blink"] + lx, ly = 0.383, 0.372 + rx, ry = 0.617, 0.372 + dx, dy = p.get("look", (0.0, 0.0)) + for cx, cy in ((lx, ly), (rx, ry)): + if p.get("cross"): + for ang in (45, -45): + a = math.radians(ang) + hx, hy = 0.042 * math.cos(a), 0.042 * math.sin(a) + d.line( + [px(cx - hx), px(cy - hy), px(cx + hx), px(cy + hy)], + fill=C_OUTLINE, + width=_w(0.014), + ) + continue + if blink > 0.55: + d.arc( + [px(cx - 0.052), px(cy - 0.030), px(cx + 0.052), px(cy + 0.040)], + start=200, + end=340, + fill=C_OUTLINE, + width=_w(0.013), + ) + continue + h = lerp(0.118, 0.030, blink) + ell(d, cx, cy, 0.106, h, C_WHITE) + if h > 0.06: + ell(d, cx + dx, cy + dy * 0.6, 0.082, min(h - 0.022, 0.092), C_IRIS, outline=None) + ell(d, cx + dx, cy + dy * 0.6, 0.046, min(h - 0.045, 0.056), C_PUPIL, outline=None) + ell(d, cx + dx - 0.020, cy + dy * 0.6 - 0.024, 0.030, 0.026, C_WHITE, outline=None) + # Tan brow dots on the slate cap (the shepherd/doberman marking) rather + # than dashes — as lines above the eyes they read as heavy eyelids and + # make an idle pet look permanently fed up. + raise_ = p.get("brow", 0.0) + angry = p.get("brow_angle", 0.0) + for side, cx in ((-1, lx), (1, rx)): + by = 0.291 - 0.020 * raise_ + ell( + d, + cx + side * 0.006, + by + side * angry * 0.020, + 0.062, + 0.040, + C_BROW, + outline=None, + ) + + +def draw_muzzle(d, p): + mouth = p["mouth"] + ell(d, 0.50, 0.487, 0.285, 0.195, C_FUR) + # nose + ell(d, 0.50, 0.440, 0.105, 0.078, C_NOSE, outline=None) + ell(d, 0.478, 0.428, 0.030, 0.020, (92, 102, 124, 255), outline=None) + if mouth > 0.02: + h = 0.030 + 0.085 * mouth + w = 0.105 + 0.055 * mouth + ell(d, 0.50, 0.500 + h / 2 - 0.008, w, h, C_NOSE) + ell(d, 0.50, 0.500 + h * 0.72, w * 0.60, h * 0.52, C_TONGUE, outline=None) + else: + # closed muzzle: a short philtrum down from the nose into two + # downward-bulging curves (PIL arcs run clockwise from 3 o'clock with + # y down, so 0->180 is the lower half — the smiling side). + d.line([px(0.50), px(0.470), px(0.50), px(0.508)], fill=C_OUTLINE, width=_w(0.011)) + for side in (-1, 1): + cx = 0.50 + side * 0.032 + d.arc( + [px(cx - 0.032), px(0.492), px(cx + 0.032), px(0.536)], + start=0, + end=180, + fill=C_OUTLINE, + width=_w(0.011), + ) + + +def draw_head(layer, p): + d = ImageDraw.Draw(layer) + draw_ears(d, p) + ell(d, HEAD_CX, HEAD_CY, HEAD_W, HEAD_H, C_FUR) + draw_cap(layer, p) + # blaze + poly(d, bolt_shape(0.50, 0.243, 0.088, 0.150), C_BOLT, outline=None) + draw_muzzle(d, p) + draw_eyes(d, p) + + +# --- extras ---------------------------------------------------------------- +def draw_extras(layer, p): + d = ImageDraw.Draw(layer) + kind = p.get("extras") + if kind == "listen": + for i in range(3): + r = 0.045 + i * 0.036 + alpha = int(210 - i * 55) + phase = p.get("phase", 0) + if (phase + i) % 3 == 0: + alpha = min(255, alpha + 45) + d.arc( + [px(0.845 - r), px(0.235 - r), px(0.845 + r), px(0.235 + r)], + start=200, + end=340, + fill=C_GLOW[:3] + (alpha,), + width=_w(0.014), + ) + elif kind == "think": + phase = p.get("phase", 0) + for i in range(3): + grow = 1.0 if i == phase % 3 else 0.62 + ell( + layer_d := d, + 0.735 + i * 0.072, + 0.145 - i * 0.030, + 0.040 * grow, + 0.040 * grow, + C_GLOW, + outline=C_OUTLINE, + ow=0.008, + ) + elif kind == "error": + poly(d, bolt_shape(0.815, 0.185, 0.070, 0.120), (235, 92, 74, 255)) + + +# --- side view: the walk cycle --------------------------------------------- +# The pose above is a front-facing sit, which is right for standing around but +# slides like a chess piece the moment the pet actually moves. Walking gets its +# own construction: a profile torso, four legs following a paw path, and a head +# side-on. Drawn facing RIGHT — ui/pet_window.py mirrors it when he walks left. + +_GROUND = 0.930 # paw centre while a foot is planted +_STRIDE = 0.088 # how far ahead of / behind the pivot a paw reaches +_LIFT = 0.080 # peak height of a paw mid-swing +_STANCE = 0.62 # fraction of the cycle a foot spends on the ground + +FRONT_PIVOT = (0.650, 0.620) +HIND_PIVOT = (0.315, 0.640) +WALK_HEAD = (0.780, 0.370, 0.260, 0.250) # cx, cy, w, h + + +def paw_position(pivot, phase): + """Where one paw is at *phase* (0..1) of the cycle. + + Stance is the half that matters: the foot is planted and travels backwards + under the dog at a constant rate. The window advances this cycle by + distance travelled rather than by clock, so that backwards travel cancels + the forward motion and the feet don't skate. + """ + phase %= 1.0 + if phase < _STANCE: + t = phase / _STANCE + return pivot[0] + _STRIDE - 2 * _STRIDE * t, _GROUND + t = (phase - _STANCE) / (1.0 - _STANCE) + return ( + pivot[0] - _STRIDE + 2 * _STRIDE * t, + _GROUND - _LIFT * math.sin(math.pi * t), + ) + + +def draw_leg(d, pivot, paw, fill, bend=0.032, top=0.052, toe=0.030): + """A limb from pivot to paw: a bezier through a displaced knee, tapered + from thigh to ankle. + + Tapering matters more than it sounds — a constant-width limb reads as a + length of white pipe, and four of them make the dog look like furniture. + """ + vx, vy = paw[0] - pivot[0], paw[1] - pivot[1] + length = math.hypot(vx, vy) or 1e-6 + nx, ny = -vy / length, vx / length # perpendicular; points backwards + knee = ( + (pivot[0] + paw[0]) / 2 + nx * bend, + (pivot[1] + paw[1]) / 2 + ny * bend, + ) + pts = [] + for i in range(13): + t = i / 12 + x = (1 - t) ** 2 * pivot[0] + 2 * (1 - t) * t * knee[0] + t**2 * paw[0] + y = (1 - t) ** 2 * pivot[1] + 2 * (1 - t) * t * knee[1] + t**2 * paw[1] + pts.append((x, y, lerp(top, toe, t))) + draw_tapered(d, pts, lambda _t: fill) + ell(d, paw[0], paw[1] + 0.008, 0.098, 0.056, fill) + + +def draw_walk_tail(d, p, dy): + """A curled plume over the back. + + Cubic rather than quadratic: a single control point can only bend one way, + which gives a straight tapered tube — a club with a white ball on the end, + not a tail. The curl back over the spine is what makes it read. + """ + wag = p["tail"] + base = (0.250, 0.575 + dy) + c1 = (0.075, 0.545 + dy - 0.030 * wag) + c2 = (0.070, 0.300 + dy - 0.040 * wag) + end = (0.215, 0.290 + dy - 0.020 * wag) + pts = [] + for i in range(29): + t = i / 28 + u = 1 - t + x = u**3 * base[0] + 3 * u**2 * t * c1[0] + 3 * u * t**2 * c2[0] + t**3 * end[0] + y = u**3 * base[1] + 3 * u**2 * t * c1[1] + 3 * u * t**2 * c2[1] + t**3 * end[1] + pts.append((x, y, lerp(0.076, 0.028, t))) + draw_tapered(d, pts, lambda t: C_DARK if t < 0.90 else C_FUR) + + +def draw_torso(d, dy): + """Rump + barrel + chest as one silhouette. + + Drawn in two passes — every shape swollen by the stroke width in the + outline colour, then every shape again at true size in the fill. Outlining + each piece individually instead leaves the construction arcs showing + across the body, which looks like the dog has panel lines. + """ + shapes = [ + ("ell", 0.300, 0.600 + dy, 0.290, 0.300, 0.0), + ("rrect", 0.480, 0.585 + dy, 0.520, 0.265, 0.130), + ("ell", 0.650, 0.590 + dy, 0.250, 0.280, 0.0), + ] + grow = 2 * OW + for colour, pad in ((C_OUTLINE, grow), (C_DARK, 0.0)): + for shape in shapes: + kind, cx, cy, w, h, extra = shape + if kind == "ell": + ell(d, cx, cy, w + pad, h + pad, colour, outline=None) + else: + rrect(d, cx, cy, w + pad, h + pad, extra + pad / 2, colour, outline=None) + # Belly kept small and low: any bigger and it merges with the cream legs + # into one white mass with a slate lid. + ell(d, 0.490, 0.672 + dy, 0.350, 0.098, C_FUR, outline=None) + + +def draw_walk_head(layer, p, dy): + d = ImageDraw.Draw(layer) + cx, cy, w, h = WALK_HEAD[0], WALK_HEAD[1] + dy, WALK_HEAD[2], WALK_HEAD[3] + + # ear first, so the head covers its base + bounce = p.get("ear_bounce", 0.0) + ear = [ + (0.690, cy - 0.030), + (0.700, cy - 0.150 - 0.012 * bounce), + (0.752, cy - 0.205 - 0.018 * bounce), + (0.788, cy - 0.090), + ] + poly(d, ear, C_DARK) + inner = [(lerp(0.735, x, 0.58), lerp(cy - 0.040, y, 0.62)) for x, y in ear] + poly(d, inner, C_INNER_EAR, outline=None) + + # neck into the chest + d.line( + [px(0.660), px(cy + 0.190), px(0.735), px(cy + 0.080)], + fill=C_OUTLINE, + width=_w(0.215), + joint="curve", + ) + d.line( + [px(0.660), px(cy + 0.190), px(0.735), px(cy + 0.080)], + fill=C_DARK, + width=_w(0.190), + joint="curve", + ) + + ell(d, cx, cy, w, h, C_FUR) + + # slate cap, clipped to the skull + mask = Image.new("L", (S, S), 0) + ImageDraw.Draw(mask).ellipse( + [px(cx - w / 2), px(cy - h / 2), px(cx + w / 2), px(cy + h / 2)], fill=255 + ) + cap = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + dc = ImageDraw.Draw(cap) + ell(dc, cx - 0.010, cy - 0.070, w * 1.02, h * 0.72, C_DARK, outline=None) + cap.putalpha(Image.composite(cap.getchannel("A"), Image.new("L", (S, S), 0), mask)) + layer.alpha_composite(cap) + + poly(d, bolt_shape(0.762, cy - 0.088, 0.062, 0.108), C_BOLT, outline=None) + + # muzzle, nose, mouth + ell(d, 0.880, cy + 0.048, 0.145, 0.108, C_FUR) + ell(d, 0.950, cy + 0.018, 0.058, 0.048, C_NOSE, outline=None) + d.arc( + [px(0.885), px(cy + 0.058), px(0.945), px(cy + 0.100)], + start=0, + end=150, + fill=C_OUTLINE, + width=_w(0.010), + ) + + # one eye in profile, plus the brow marking + ell(d, 0.812, cy - 0.020, 0.092, 0.100, C_WHITE) + ell(d, 0.820, cy - 0.020, 0.062, 0.070, C_IRIS, outline=None) + ell(d, 0.824, cy - 0.020, 0.036, 0.042, C_PUPIL, outline=None) + ell(d, 0.812, cy - 0.040, 0.026, 0.022, C_WHITE, outline=None) + ell(d, 0.795, cy - 0.088, 0.055, 0.034, C_BROW, outline=None) + + # Collar: a band *across* the neck, so it has to run perpendicular to it. + # Along the neck it just reads as an orange brick stuck to his chest. + collar = [ + (px(0.648), px(cy + 0.098)), + (px(0.762), px(cy + 0.196)), + ] + d.line(collar, fill=C_OUTLINE, width=_w(0.070), joint="curve") + d.line(collar, fill=C_COLLAR, width=_w(0.050), joint="curve") + ell(d, 0.712, cy + 0.196, 0.070, 0.070, C_TAG) + poly(d, bolt_shape(0.712, cy + 0.196, 0.025, 0.044), C_OUTLINE, outline=None) + + +def render_walk_frame(p) -> Image.Image: + base = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + d = ImageDraw.Draw(base) + phase = p["phase"] + # Two contacts per cycle, so the body dips twice — the give-away that a + # walk cycle is weight-bearing rather than a slide. + dy = -0.011 * abs(math.sin(2 * math.pi * phase)) + head_dy = dy * 0.6 - 0.006 * math.sin(2 * math.pi * phase + 0.7) + + # Diagonal pairs (a trot): each front leg moves with the opposite hind. + far_front = paw_position(FRONT_PIVOT, phase + 0.5) + far_hind = paw_position(HIND_PIVOT, phase) + near_front = paw_position(FRONT_PIVOT, phase) + near_hind = paw_position(HIND_PIVOT, phase + 0.5) + + draw_walk_tail(d, p, dy) + # far side first, in the shade colour, so the near legs read as in front + draw_leg(d, (HIND_PIVOT[0], HIND_PIVOT[1] + dy), far_hind, C_FUR_FAR, bend=0.046) + draw_leg(d, (FRONT_PIVOT[0], FRONT_PIVOT[1] + dy), far_front, C_FUR_FAR) + + draw_torso(d, dy) + + draw_leg(d, (HIND_PIVOT[0], HIND_PIVOT[1] + dy), near_hind, C_FUR, bend=0.046) + draw_leg(d, (FRONT_PIVOT[0], FRONT_PIVOT[1] + dy), near_front, C_FUR) + + draw_walk_head(base, p, head_dy) + return base.resize((OUT, OUT), Image.LANCZOS) + + +# --- frame assembly -------------------------------------------------------- +def default_pose(**over): + p = dict( + breathe=0.0, + tail=0.0, + ear=0.0, + ear_twitch=0.0, + blink=0.0, + mouth=0.0, + tilt=0.0, + head_dy=0.0, + look=(0.0, 0.0), + brow=0.0, + brow_angle=0.0, + cross=False, + tag_glow=False, + extras=None, + phase=0, + ) + p.update(over) + return p + + +def render_frame(p) -> Image.Image: + if p.get("pose") == "walk": + return render_walk_frame(p) + base = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + + body = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + db = ImageDraw.Draw(body) + draw_tail(db, p) + draw_body(db, p) + draw_collar(db, p) + base.alpha_composite(body) + + head = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + draw_head(head, p) + if p["tilt"]: + head = head.rotate( + p["tilt"], resample=Image.BICUBIC, center=(px(HEAD_CX), px(NECK_Y)) + ) + dy = int(px(p["head_dy"])) + if dy: + shifted = Image.new("RGBA", (S, S), (0, 0, 0, 0)) + shifted.alpha_composite(head, (0, dy)) + head = shifted + base.alpha_composite(head) + + draw_extras(base, p) + return base.resize((OUT, OUT), Image.LANCZOS) + + +def frames_for(state: str) -> list[dict]: + if state == "idle": + out = [] + for i in range(8): + t = i / 8 + br = math.sin(t * 2 * math.pi) + out.append( + default_pose( + breathe=br, + head_dy=-0.006 * br, + tail=math.sin(t * 4 * math.pi), + blink=1.0 if i == 6 else 0.0, + ) + ) + return out + if state == "listening": + out = [] + for i in range(4): + t = i / 4 + out.append( + default_pose( + ear=1.0, + ear_twitch=0.35 * math.sin(t * 2 * math.pi), + tilt=-7 + 2.0 * math.sin(t * 2 * math.pi), + brow=1.0, + tail=0.5 * math.sin(t * 2 * math.pi), + head_dy=-0.008, + tag_glow=True, + extras="listen", + phase=i, + ) + ) + return out + if state == "thinking": + out = [] + for i in range(6): + t = i / 6 + out.append( + default_pose( + ear=0.25, + tilt=6.0, + look=(0.022, -0.026), + brow=0.5, + breathe=0.4 * math.sin(t * 2 * math.pi), + tail=0.2 * math.sin(t * 2 * math.pi), + extras="think", + phase=i // 2, + ) + ) + return out + if state == "talking": + out = [] + for i in range(4): + t = i / 4 + open_ = (math.sin(t * 2 * math.pi) + 1) / 2 + out.append( + default_pose( + mouth=0.25 + 0.75 * open_, + ear=0.6, + head_dy=-0.010 * open_, + breathe=open_, + tail=math.sin(t * 2 * math.pi + 1.0), + brow=0.35, + ) + ) + return out + if state == "walk": + # 8 frames: two full strides, so the loop lands back on the pose it + # started from and the cycle is seamless however it's entered. + out = [] + for i in range(8): + phase = i / 8 + out.append( + default_pose( + pose="walk", + phase=phase, + tail=math.sin(2 * math.pi * phase), + ear_bounce=math.sin(2 * math.pi * phase + 0.9), + ) + ) + return out + if state == "error": + return [ + default_pose(ear=-1.0, cross=True, brow_angle=1.0, mouth=0.35, tail=-0.6, + extras="error"), + default_pose(ear=-0.85, cross=True, brow_angle=1.0, mouth=0.15, tail=-0.4, + head_dy=0.008), + ] + raise ValueError(state) + + +STATES = ["idle", "listening", "thinking", "talking", "error", "walk"] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--out", + type=Path, + default=Path(__file__).resolve().parent.parent / "bolt_pet" / "assets" / "sprites", + ) + ap.add_argument("--states", nargs="*", default=STATES) + args = ap.parse_args() + + for state in args.states: + d = args.out / state + d.mkdir(parents=True, exist_ok=True) + for old in d.glob("*.png"): + old.unlink() + for i, pose in enumerate(frames_for(state)): + render_frame(pose).save(d / f"frame_{i:02d}.png") + print(f"{state}: {len(frames_for(state))} frames -> {d}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_monitors.py b/tests/test_monitors.py new file mode 100644 index 0000000..b9efcf4 --- /dev/null +++ b/tests/test_monitors.py @@ -0,0 +1,169 @@ +"""Screen layout logic — resolving `petctl jump` targets and describing the +setup. Pure: the monitor list is normally published by the UI, so none of this +needs a display.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import monitors as m + + +def grid(): + """The 2x2 setup this was built against: four 1080p screens. + + [1 HDMI-0] [2 HDMI-1] + [3 DP-0 ] [4 DP-2 ] + """ + return [ + m.Monitor(0, "HDMI-0", 0, 0, 1920, 1080, primary=False), + m.Monitor(1, "HDMI-1", 1920, 0, 1920, 1080, primary=False), + m.Monitor(2, "DP-0", 0, 1080, 1920, 1080, primary=True), + m.Monitor(3, "DP-2", 1920, 1080, 1920, 1080, primary=False), + ] + + +def two(): + return [ + m.Monitor(0, "eDP-1", 0, 0, 1920, 1080, primary=True), + m.Monitor(1, "HDMI-1", 1920, 0, 2560, 1440), + ] + + +def test_numbers_shown_to_humans_are_one_based(): + left, right = two() + assert left.index == 0 and left.number == 1 + assert right.index == 1 and right.number == 2 + assert "1: eDP-1 1920x1080 (primary)" == left.label + + +def test_geometry_helpers(): + screen = m.Monitor(1, "HDMI-1", 1920, 0, 1920, 1080) + assert screen.right == 3840 and screen.bottom == 1080 + assert screen.center == (2880, 540) + assert screen.contains(1920, 0) + assert screen.contains(3839, 1079) + assert not screen.contains(3840, 0) # right edge is exclusive + assert not screen.contains(1919, 0) + + +def test_monitor_containing_and_nearest(): + screens = grid() + assert m.monitor_containing(screens, 100, 100).name == "HDMI-0" + assert m.monitor_containing(screens, 2000, 1500).name == "DP-2" + assert m.monitor_containing(screens, -50, -50) is None + # off the desktop entirely still resolves to something + assert m.nearest_monitor(screens, -500, -500).name == "HDMI-0" + + +def test_resolve_by_number(): + screens = grid() + assert m.resolve(screens, "3").name == "DP-0" + with pytest.raises(ValueError, match="no monitor 9"): + m.resolve(screens, "9") + with pytest.raises(ValueError): + m.resolve(screens, "0") + + +def test_resolve_next_and_prev_wrap(): + screens = grid() + assert m.resolve(screens, "next", current=3).number == 1 + assert m.resolve(screens, "prev", current=0).number == 4 + assert m.resolve(screens, "next", current=0).number == 2 + + +def test_resolve_primary_and_other(): + screens = grid() + assert m.resolve(screens, "primary", current=0).name == "DP-0" + # "other" on a two-screen setup is genuinely the other one + pair = two() + assert m.resolve(pair, "other", current=0).number == 2 + assert m.resolve(pair, "other", current=1).number == 1 + + +def test_resolve_directions_on_a_grid(): + screens = grid() + # from top-left (HDMI-0) + assert m.resolve(screens, "right", current=0).name == "HDMI-1" + assert m.resolve(screens, "down", current=0).name == "DP-0" + # from bottom-right (DP-2) + assert m.resolve(screens, "left", current=3).name == "DP-0" + assert m.resolve(screens, "up", current=3).name == "HDMI-1" + + +def test_direction_prefers_the_best_aligned_screen(): + screens = grid() + # "right" from DP-0 (bottom-left) must pick DP-2 (same row), not HDMI-1, + # even though both are to the right. + assert m.resolve(screens, "right", current=2).name == "DP-2" + + +def test_resolve_direction_with_nothing_there(): + screens = grid() + with pytest.raises(ValueError, match="no monitor to the left"): + m.resolve(screens, "left", current=0) + + +def test_resolve_by_name_is_fuzzy_but_refuses_ambiguity(): + screens = grid() + assert m.resolve(screens, "dp-2").name == "DP-2" + assert m.resolve(screens, "HDMI-0").name == "HDMI-0" + with pytest.raises(ValueError, match="matches several"): + m.resolve(screens, "hdmi") + + +def test_resolve_unknown_spec_lists_the_options(): + screens = two() + with pytest.raises(ValueError) as excinfo: + m.resolve(screens, "the big one") + assert "eDP-1" in str(excinfo.value) and "HDMI-1" in str(excinfo.value) + + +def test_resolve_without_a_current_screen_falls_back_to_primary(): + screens = grid() + # primary is index 2, so "next" from nowhere is index 3 + assert m.resolve(screens, "next", current=None).number == 4 + # an out-of-range current is treated the same way rather than exploding + assert m.resolve(screens, "next", current=99).number == 4 + + +def test_resolve_needs_monitors(): + with pytest.raises(ValueError, match="no monitors"): + m.resolve([], "next") + with pytest.raises(ValueError, match="needs a target"): + m.resolve(grid(), "") + + +def test_random_always_moves_somewhere_else(): + screens = grid() + for current in range(4): + assert m.resolve(screens, "random", current=current).index != current + + +def test_summary_is_one_line_and_marks_where_the_pet_is(): + line = m.summary(grid(), current=1) + assert "\n" not in line + assert line.startswith("4 monitors:") + assert "Bolt is on 2" in line + assert m.summary([]) is None + assert "Bolt is on" not in m.summary(grid(), current=None) + + +def test_annotate_matches_the_screen_context_style(): + out = m.annotate("what's on the other screen?", grid(), current=0) + assert out.startswith("what's on the other screen?") + assert "[4 monitors:" in out + # nothing to say, nothing added + assert m.annotate("hello", [], None) == "hello" + assert m.annotate("", grid(), 0) == "" + + +def test_describe_lists_every_screen_and_flags_the_pet(): + text = m.describe(grid(), current=2) + assert text.count("\n") == 4 # header + 4 screens + assert "DP-0" in text and "+0+1080" in text + assert text.count("Bolt is here") == 1 + assert m.describe([]) == "[pet] no monitor information available" diff --git a/tests/test_pet_window_features.py b/tests/test_pet_window_features.py index 859dd60..76b440e 100644 --- a/tests/test_pet_window_features.py +++ b/tests/test_pet_window_features.py @@ -15,7 +15,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from bolt_pet import config from bolt_pet.state import PetState -from bolt_pet.ui.pet_window import _EMOTE_TICKS, PetWindow, emote_transform +from bolt_pet.ui.pet_window import ( + _EMOTE_TICKS, _WALK_PIXELS_PER_FRAME, PetWindow, emote_transform, +) +from bolt_pet.ui.sprite import WALK @pytest.fixture(scope="module") @@ -179,3 +182,153 @@ def test_click_through_toggles_mouse_transparency(pet): pet.set_click_through(False) assert pet.testAttribute(Qt.WA_TransparentForMouseEvents) is False + + +# ── monitors ──────────────────────────────────────────────────────────────── + +def test_window_publishes_a_monitor_list(pet): + """Whatever the test host's screen setup is, the window must describe it + in the shape the controller expects.""" + monitors = pet.monitors() + assert monitors, "offscreen Qt still reports at least one screen" + assert [m.index for m in monitors] == list(range(len(monitors))) + assert all(m.width > 0 and m.height > 0 for m in monitors) + assert all(m.name for m in monitors) + assert sum(1 for m in monitors if m.primary) <= 1 + + +def test_publish_monitors_re_emits_when_forced(pet): + """ui/app.py relies on this: the window is built before the controller + exists, so its constructor's publish reaches nobody and has to be redone.""" + seen = [] + pet.monitors_changed.connect(seen.append) + pet.publish_monitors() # force defaults to True + assert len(seen) == 1 + pet.publish_monitors(force=False) # nothing changed -> stays quiet + assert len(seen) == 1 + + +def test_pet_reports_which_monitor_it_is_on(pet): + seen = [] + pet.pet_monitor_changed.connect(seen.append) + pet.publish_monitors() + assert seen and seen[-1] == pet.current_monitor_index() + assert 0 <= seen[-1] < len(pet.monitors()) + + +def test_jump_moves_the_window_onto_the_target_screen(pet): + monitors = pet.monitors() + target = len(monitors) - 1 + pet.apply_action({"action": "jump", "monitor": target}) + assert pet.current_monitor_index() == target + # a jump lands with a hop rather than sliding there + assert pet._emote == "hop" + + +def test_jump_cancels_a_stroll_so_it_does_not_walk_back(pet): + pet.apply_action({"action": "move", "anchor": "top-left"}) + assert pet._wander_target is not None + pet.apply_action({"action": "jump", "monitor": 0}) + assert pet._wander_target is None + assert pet._commanded_move is False + + +def test_jump_to_a_bogus_index_is_a_no_op(pet): + before = pet.pos() + pet.apply_action({"action": "jump", "monitor": 99}) + pet.apply_action({"action": "jump", "monitor": -1}) + assert pet.pos() == before + + +# ── walk cycle ────────────────────────────────────────────────────────────── + +def test_walk_art_loads_as_a_non_state_animation(pet): + """Walking is a property of movement, not a PetState, so it lives outside + the state machine but still loads like any other animation.""" + assert pet.sprites.has(WALK) + assert len(pet.sprites.get(WALK).frames) == 8 + assert pet.sprites.get("nonsense") is pet.sprites.get(PetState.IDLE) + assert not pet.sprites.has("nonsense") + + +def test_walking_overrides_the_state_animation(pet): + assert pet._animation_key() == pet._current_state + pet._advance_walk(50, 0, 1.0) + assert pet._animation_key() == WALK + + +def test_walk_cycle_advances_by_distance_not_by_the_clock(pet): + """The planted paw tracks backwards at the speed the window moves + forwards; drive it off the animation timer instead and the feet skate.""" + anim = pet.sprites.get(WALK) + anim.reset() + pet._advance_walk(100, 0, _WALK_PIXELS_PER_FRAME * 3) + assert anim._index == 3 + # a step too small to cross the threshold banks the distance instead + pet._advance_walk(100, 0, _WALK_PIXELS_PER_FRAME * 0.5) + assert anim._index == 3 + pet._advance_walk(100, 0, _WALK_PIXELS_PER_FRAME * 0.5) + assert anim._index == 4 + + +def test_the_animation_timer_does_not_double_step_the_walk(pet): + anim = pet.sprites.get(WALK) + pet._advance_walk(50, 0, 1.0) + anim.reset() + pet._advance_frame() + assert anim._index == 0 + + +def test_facing_follows_horizontal_travel(pet): + pet._advance_walk(50, 0, 1.0) + assert pet._facing == 1 + pet._advance_walk(-50, 0, 1.0) + assert pet._facing == -1 + + +def test_a_near_vertical_stroll_does_not_flip_him(pet): + """Rounding noise on dx would otherwise flip him back and forth every + tick on a straight-up walk.""" + pet._facing = 1 + pet._advance_walk(0.4, 60, 1.0) + assert pet._facing == 1 + + +def test_walking_left_paints_a_mirrored_frame(pet): + frame = pet.sprites.get(WALK).current() + pet._facing = 1 + assert pet._oriented(frame) is frame # art is drawn facing right + pet._facing = -1 + flipped = pet._oriented(frame) + assert flipped is not frame + assert flipped.size() == frame.size() + assert pet._oriented(frame) is flipped # cached, not re-flipped per paint + + +def test_stopping_resets_the_cycle_to_a_standing_frame(pet): + pet._advance_walk(50, 0, _WALK_PIXELS_PER_FRAME * 2) + assert pet._walking + pet._stop_walking() + assert not pet._walking + assert pet._walk_distance == 0.0 + assert pet.sprites.get(WALK)._index == 0 + assert pet._animation_key() == pet._current_state + + +def test_walk_art_suppresses_the_hard_coded_bob(pet): + """The frames carry their own weight shift — bobbing the window as well + would double it up.""" + pet._advance_walk(50, 0, 5.0) + assert pet._bob_offset == 0 + + +def test_without_walk_art_it_falls_back_to_the_old_bob(qt_app, tmp_path): + window = PetWindow(sprite_dir=tmp_path) + try: + assert not window.sprites.has(WALK) + window._advance_walk(50, 0, 5.0) + assert window._walking + assert window._animation_key() == window._current_state + assert window._bob_offset < 0 # still visibly moving + finally: + window.close() diff --git a/tests/test_screen_text.py b/tests/test_screen_text.py new file mode 100644 index 0000000..88a26e1 --- /dev/null +++ b/tests/test_screen_text.py @@ -0,0 +1,168 @@ +"""OCR plumbing for `petctl read` — engine selection and output cleanup. + +Only the pure half is covered, per the testing conventions: capture and the +OCR call itself need a real screen and a real engine. Engine probes are +injected so these pass on a machine with a different set installed (or none). +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from bolt_pet import screen_text +from bolt_pet.monitors import Monitor + + +def probes(modules=(), binaries=()): + return (lambda name: name in modules), ( + lambda name: f"/usr/bin/{name}" if name in binaries else None + ) + + +def test_prefers_tesseract_when_fully_installed(): + has_module, which = probes({"pytesseract"}, {"tesseract"}) + assert screen_text.resolve_engine(has_module, which) == ("pytesseract", "") + + +def test_falls_back_to_rapidocr_when_tesseract_binary_is_absent(): + has_module, which = probes({"pytesseract", "rapidocr_onnxruntime"}, set()) + engine, reason = screen_text.resolve_engine(has_module, which) + assert engine == "rapidocr" + assert reason == "" + + +def test_pytesseract_without_the_binary_says_which_half_is_missing(): + """The commonest broken setup: `pip install pytesseract` and stop, not + realising the actual engine is a system package.""" + has_module, which = probes({"pytesseract"}, set()) + engine, reason = screen_text.resolve_engine(has_module, which) + assert engine is None + assert "tesseract binary" in reason + assert "apt install tesseract-ocr" in reason + + +def test_nothing_installed_explains_how_to_fix_it(): + has_module, which = probes(set(), set()) + engine, reason = screen_text.resolve_engine(has_module, which) + assert engine is None + assert "pip install" in reason + + +def test_capture_availability_follows_mss(): + assert screen_text.capture_available(lambda name: name == "mss") + assert not screen_text.capture_available(lambda name: False) + + +def test_clean_drops_ocr_noise_and_blank_runs(): + raw = "Firefox\n\n\n |\n .\nBuild failed\n~\n" + assert screen_text.clean_ocr_text(raw) == "Firefox\nBuild failed" + + +def test_clean_collapses_whitespace_but_keeps_line_structure(): + raw = " File edit view \nline\ttwo " + assert screen_text.clean_ocr_text(raw) == "File edit view\nline two" + + +def test_clean_drops_consecutive_duplicates_only(): + raw = "Terminal\nTerminal\nEditor\nTerminal" + assert screen_text.clean_ocr_text(raw) == "Terminal\nEditor\nTerminal" + + +def test_clean_keeps_short_but_real_tokens(): + # two alphanumerics is the bar — "ok" and "42" survive, "-" doesn't + assert screen_text.clean_ocr_text("ok\n-\n42") == "ok\n42" + + +def test_clean_truncates_and_says_so(): + out = screen_text.clean_ocr_text("word " * 500, max_chars=100) + assert out.endswith("[truncated]") + # the cap applies to the text, before the marker is appended + assert len(out.split("\n[truncated]")[0]) <= 100 + + +def test_clean_handles_empty_input(): + assert screen_text.clean_ocr_text("") == "" + assert screen_text.clean_ocr_text(None) == "" + + +def test_format_reading_names_the_monitor(): + monitor = Monitor(1, "HDMI-1", 1920, 0, 1920, 1080) + out = screen_text.format_reading(monitor, "Build failed") + assert "monitor 2 (HDMI-1)" in out + assert out.endswith("Build failed") + + +def test_format_reading_when_nothing_was_recognised(): + monitor = Monitor(0, "DP-0", 0, 0, 1920, 1080) + assert "no text recognised" in screen_text.format_reading(monitor, " ") + + +def test_read_monitor_never_raises_without_an_engine(monkeypatch): + """Its return value goes back to the server as command output, so every + failure has to come back as a sentence rather than an exception.""" + monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True) + monkeypatch.setattr( + screen_text, "resolve_engine", lambda *a, **k: (None, "no engine here") + ) + out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080)) + assert out.startswith("[pet]") + assert "no engine here" in out + + +def test_read_monitor_reports_a_failed_capture(monkeypatch): + monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True) + monkeypatch.setattr(screen_text, "resolve_engine", lambda *a, **k: ("pytesseract", "")) + monkeypatch.setattr(screen_text, "capture", lambda monitor: None) + out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080)) + assert "couldn't capture" in out and "Wayland" in out + + +def test_read_monitor_survives_an_exploding_engine(monkeypatch): + monkeypatch.setattr(screen_text, "capture_available", lambda *a, **k: True) + monkeypatch.setattr(screen_text, "resolve_engine", lambda *a, **k: ("pytesseract", "")) + monkeypatch.setattr(screen_text, "capture", lambda monitor: object()) + monkeypatch.setattr( + screen_text, "_ocr", lambda image, engine: (_ for _ in ()).throw(RuntimeError("boom")) + ) + out = screen_text.read_monitor(Monitor(0, "DP-0", 0, 0, 1920, 1080)) + assert "OCR failed" in out and "boom" in out + + +def test_read_monitors_splits_the_budget(monkeypatch): + seen = [] + + def fake_read(monitor, max_chars): + seen.append((monitor.number, max_chars)) + return f"screen {monitor.number}" + + monkeypatch.setattr(screen_text, "read_monitor", fake_read) + screens = [ + Monitor(0, "A", 0, 0, 100, 100), + Monitor(1, "B", 100, 0, 100, 100), + Monitor(2, "C", 200, 0, 100, 100), + ] + out = screen_text.read_monitors(screens, 3000) + assert [n for n, _ in seen] == [1, 2, 3] + assert all(limit == 1000 for _, limit in seen) + assert out.count("screen ") == 3 + + +def test_read_monitors_keeps_a_floor_on_the_budget(monkeypatch): + monkeypatch.setattr( + screen_text, "read_monitor", lambda monitor, max_chars: str(max_chars) + ) + screens = [Monitor(i, str(i), 0, 0, 10, 10) for i in range(20)] + # 100/20 would be 5 characters per screen, which is useless — floor wins + assert "400" in screen_text.read_monitors(screens, 100) + + +def test_read_monitors_with_one_screen_uses_the_whole_budget(monkeypatch): + monkeypatch.setattr( + screen_text, "read_monitor", lambda monitor, max_chars: str(max_chars) + ) + assert screen_text.read_monitors([Monitor(0, "A", 0, 0, 10, 10)], 4000) == "4000" + + +def test_read_monitors_with_no_screens(): + assert "no monitor information" in screen_text.read_monitors([], 4000) diff --git a/tests/test_screens_feature.py b/tests/test_screens_feature.py new file mode 100644 index 0000000..15f272a --- /dev/null +++ b/tests/test_screens_feature.py @@ -0,0 +1,233 @@ +"""End-to-end wiring for the multi-monitor features: `petctl jump`, `petctl +monitors`, `petctl read`, and the screen-layout note that rides along with +each utterance. + +Needs a QApplication (signals), so run with QT_QPA_PLATFORM=offscreen. +""" + +import sys +from pathlib import Path + +import pytest +from PySide6.QtWidgets import QApplication + +from bolt_pet import controller as controller_mod +from bolt_pet import pet_actions +from bolt_pet.monitors import Monitor + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +_app = QApplication.instance() or QApplication(["test"]) + + +@pytest.fixture(autouse=True) +def no_screen_probes(monkeypatch): + monkeypatch.setattr(controller_mod.screen_context, "is_fullscreen_active", lambda: False) + monkeypatch.setattr(controller_mod.screen_context, "context_for", lambda text: text) + + +@pytest.fixture +def ctrl(): + controller = controller_mod.PetController() + controller.set_monitors( + [ + Monitor(0, "HDMI-0", 0, 0, 1920, 1080), + Monitor(1, "HDMI-1", 1920, 0, 1920, 1080), + Monitor(2, "DP-0", 0, 1080, 1920, 1080, primary=True), + ] + ) + controller.set_pet_monitor(0) + return controller + + +def _capture(signal): + events = [] + signal.connect(lambda *a: events.append(a[0] if len(a) == 1 else a)) + return events + + +# ── parsing ───────────────────────────────────────────────────────────────── + +def test_jump_parses_without_validating_the_target(): + """Which monitors exist is a runtime fact, so the pure parser passes the + spec through and monitors.resolve() judges it later.""" + assert pet_actions.parse("petctl jump 2") == {"action": "jump", "target": "2"} + assert pet_actions.parse("petctl jump next") == {"action": "jump", "target": "next"} + assert pet_actions.parse("petctl monitor left") == {"action": "jump", "target": "left"} + assert pet_actions.parse("petctl screen HDMI-1") == {"action": "jump", "target": "HDMI-1"} + # a nonsense target is still parsed — it fails at resolve time, with a + # message listing the real monitors + assert pet_actions.parse("petctl jump sideways") == { + "action": "jump", "target": "sideways", + } + + +def test_jump_needs_a_target(): + with pytest.raises(pet_actions.ActionError, match="needs a monitor"): + pet_actions.parse("petctl jump") + + +def test_read_defaults_to_the_current_screen(): + assert pet_actions.parse("petctl read") == {"action": "read", "target": "here"} + assert pet_actions.parse("petctl read all") == {"action": "read", "target": "all"} + assert pet_actions.parse("petctl look 2") == {"action": "read", "target": "2"} + assert pet_actions.parse("petctl see here") == {"action": "read", "target": "here"} + + +def test_monitors_verb(): + for spelling in ("monitors", "screens", "displays"): + assert pet_actions.parse(f"petctl {spelling}") == {"action": "monitors"} + + +def test_help_mentions_the_new_verbs(): + assert "petctl jump" in pet_actions.HELP + assert "petctl read" in pet_actions.HELP + assert "petctl monitors" in pet_actions.HELP + + +# ── jump ──────────────────────────────────────────────────────────────────── + +def test_jump_resolves_to_an_index_before_reaching_the_window(ctrl): + """The controller resolves and emits a concrete index, so the window can't + re-resolve the spec against a different screen ordering.""" + actions = _capture(ctrl.action) + out = ctrl._handle_command("petctl jump next") + assert actions == [{"action": "jump", "monitor": 1}] + assert "monitor 2: HDMI-1" in out + + +def test_jump_by_direction_uses_the_published_layout(ctrl): + actions = _capture(ctrl.action) + ctrl._handle_command("petctl jump down") + assert actions == [{"action": "jump", "monitor": 2}] + + +def test_jump_tracks_where_the_pet_actually_is(ctrl): + ctrl.set_pet_monitor(1) + actions = _capture(ctrl.action) + ctrl._handle_command("petctl jump left") + assert actions == [{"action": "jump", "monitor": 0}] + + +def test_jump_to_a_nonexistent_monitor_reports_back_and_moves_nothing(ctrl): + actions = _capture(ctrl.action) + out = ctrl._handle_command("petctl jump 7") + assert actions == [] + assert "no monitor 7" in out + assert "you have 3" in out + + +def test_jump_never_reaches_the_shell(monkeypatch, ctrl): + ran = [] + monkeypatch.setattr(controller_mod.server_client, "run_local_command", ran.append) + ctrl._handle_command("petctl jump 2") + ctrl._handle_command("petctl monitors") + ctrl._handle_command("petctl read") + assert ran == [] + + +# ── monitors ──────────────────────────────────────────────────────────────── + +def test_monitors_lists_the_layout_and_where_the_pet_is(ctrl): + ctrl.set_pet_monitor(2) + out = ctrl._handle_command("petctl monitors") + assert "3 monitor(s)" in out + assert "HDMI-0" in out and "DP-0" in out + assert out.count("Bolt is here") == 1 + + +def test_monitors_before_the_ui_has_published_anything(): + fresh = controller_mod.PetController() + assert "no monitor information" in fresh._handle_command("petctl monitors") + + +# ── read ──────────────────────────────────────────────────────────────────── + +def test_read_here_uses_the_pets_own_screen(monkeypatch, ctrl): + seen = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitor", + lambda monitor, limit: seen.append(monitor.number) or "text", + ) + ctrl.set_pet_monitor(1) + assert ctrl._handle_command("petctl read") == "text" + assert seen == [2] + + +def test_read_a_named_screen(monkeypatch, ctrl): + seen = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitor", + lambda monitor, limit: seen.append(monitor.name) or "text", + ) + ctrl._handle_command("petctl read DP-0") + assert seen == ["DP-0"] + + +def test_read_all_goes_through_the_multi_screen_path(monkeypatch, ctrl): + seen = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitors", + lambda monitors, limit: seen.append(len(monitors)) or "everything", + ) + assert ctrl._handle_command("petctl read all") == "everything" + assert seen == [3] + + +def test_read_an_unknown_screen_explains_rather_than_raising(ctrl): + out = ctrl._handle_command("petctl read 9") + assert out.startswith("[pet]") + assert "no monitor 9" in out + + +def test_read_respects_the_kill_switch(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "SCREEN_TEXT", False) + called = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitor", + lambda *a, **k: called.append(1) or "text", + ) + out = ctrl._handle_command("petctl read") + assert called == [] + assert "disabled" in out and "SCREEN_TEXT" in out + + +def test_read_passes_the_character_cap_through(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "SCREEN_TEXT_MAX_CHARS", 123) + seen = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitor", + lambda monitor, limit: seen.append(limit) or "text", + ) + ctrl._handle_command("petctl read") + assert seen == [123] + + +# ── per-turn context ──────────────────────────────────────────────────────── + +def test_layout_rides_along_with_each_utterance(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", True) + out = ctrl._with_context("what's on the other screen?") + assert out.startswith("what's on the other screen?") + assert "3 monitors:" in out + assert "Bolt is on 1" in out + + +def test_layout_context_can_be_switched_off(monkeypatch, ctrl): + monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", False) + assert ctrl._with_context("hello") == "hello" + + +def test_screen_text_never_rides_along_automatically(monkeypatch, ctrl): + """The layout is free; the *contents* cost an OCR pass and a lot of + privacy, so they only ever move on an explicit petctl read.""" + monkeypatch.setattr(controller_mod.config, "MONITOR_CONTEXT", True) + called = [] + monkeypatch.setattr( + controller_mod.screen_text, "read_monitor", lambda *a, **k: called.append(1) + ) + monkeypatch.setattr( + controller_mod.screen_text, "read_monitors", lambda *a, **k: called.append(1) + ) + ctrl._with_context("hello") + assert called == []