TheMajesticNetwork/Bolt-Pet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Multi-monitor jumps, screen OCR, and generated sprite art

Browse Source 
petctl gains screen verbs: `jump` (1-based number, name, next/prev/
primary/other, or a direction resolved from real geometry), `monitors`,
and `read` for OCR of a monitor's contents.

- monitors.py: pure layout model + jump-target resolution. The monitor
  list is published by PetWindow from QGuiApplication.screens() over a
  queued signal, so the controller and window agree on what "monitor 2"
  means; xrandr and Qt order screens differently on the same machine.
- screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR).
  Nothing captures unless the server asks, and the text rides back up
  the tool-result relay so Bolt can read a screen mid-turn. Both deps
  optional, soft-failing with a reason. SCREEN_TEXT=false removes it.
- Query verbs are answered in controller._handle_command rather than
  pet_actions.describe(), because their output is the point.
- scripts/generate_bolt_sprites.py draws every frame; walk/ is a
  side-view cycle stepped by distance travelled, not by the animation
  timer, so the planted paw tracks the window exactly. sprite.py loads
  it via EXTRA_ANIMATIONS keyed by name, with has() so callers can
  decline a placeholder blob.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Aaron Franks
2026-07-28 16:17:11 -06:00
parent ccb3aeb7ae
commit b121bbba17
50 changed files with 2460 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bolt_pet/ui/app.py
+7
View File
@@ -46,6 +46,13 @@ def run() -> int:
controller.finished.connect(thread.quit)
window.talk_requested.connect(controller.request_talk_now)
# The window owns the screen list and tells the controller about it, so
# both ends agree on what "monitor 2" means (see monitors.py).
window.monitors_changed.connect(controller.set_monitors)
window.pet_monitor_changed.connect(controller.set_pet_monitor)
# PetWindow publishes once in its constructor, which ran before those
# connections existed — so say it again now that anyone is listening.
window.publish_monitors()
window.copied.connect(lambda text: _log(f"Copied to clipboard: {text[:60]}"))
history_window = HistoryWindow(controller.history)
bolt_pet/ui/pet_window.py
+188 -5
View File
@@ -20,8 +20,9 @@ from PySide6.QtGui import (
from PySide6.QtWidgets import QApplication, QWidget
from .. import config
from ..monitors import Monitor
from ..state import PetState
from .sprite import SpriteSet
from .sprite import WALK, SpriteSet
_DRAG_THRESHOLD_PX = 4
# Movement runs on its own ~30fps timer, independent of the (slower) sprite
@@ -29,6 +30,12 @@ _DRAG_THRESHOLD_PX = 4
_WANDER_TICK_MS = 33
_EMOTE_TICKS = 36 # ~1.2s per emote at the tick rate above
_NAP_OPACITY = 0.35
# How far the pet travels per walk-cycle frame. The cycle is advanced by
# distance rather than by the animation clock so a planted paw tracks backwards
# at exactly the speed the window moves forwards — drive it off a timer instead
# and the feet skate whenever PET_WANDER_SPEED doesn't happen to match the fps.
# Eight frames at 13px is a ~104px stride cycle, a bit under the pet's width.
_WALK_PIXELS_PER_FRAME = 13.0
def emote_transform(emote: str, progress: float) -> tuple[float, float, float, float]:
@@ -164,6 +171,12 @@ class SpeechBubble(QWidget):
class PetWindow(QWidget):
talk_requested = Signal()
copied = Signal(str) # bubble text the user just put on the clipboard
# The screen layout, published *to* the controller (queued, cross-thread).
# The window is the only thing allowed to ask Qt about screens, so the
# controller and the window can never disagree about what "monitor 2"
# means — see monitors.py.
monitors_changed = Signal(list) # list[monitors.Monitor]
pet_monitor_changed = Signal(int) # 0-based index the pet is standing on
def __init__(self, sprite_dir: Optional[Path] = None, size: Optional[int] = None):
super().__init__()
@@ -201,6 +214,10 @@ class PetWindow(QWidget):
self._next_wander_at = 0.0
self._bob_offset = 0
self._bob_phase = 0.0
self._walking = False
self._facing = 1 # +1 right, -1 left; the walk art is drawn facing right
self._walk_distance = 0.0
self._mirror_cache: dict[int, QPixmap] = {}
self._schedule_next_wander()
self._wander_timer = QTimer(self)
self._wander_timer.timeout.connect(self._movement_tick)
@@ -210,6 +227,18 @@ class PetWindow(QWidget):
self.set_click_through(config.PET_CLICK_THROUGH)
self._place_start_position()
self._monitors: list[Monitor] = []
self._pet_monitor: Optional[int] = None
self._last_published_pos: Optional[QPoint] = None
app = QApplication.instance()
if app is not None:
# Screens come and go — a laptop docking, a TV waking up. Republish
# rather than letting Bolt jump to a monitor that's been unplugged.
app.screenAdded.connect(lambda _s: self.publish_monitors())
app.screenRemoved.connect(lambda _s: self.publish_monitors())
app.primaryScreenChanged.connect(lambda _s: self.publish_monitors())
self.publish_monitors()
# ── placement ────────────────────────────────────────────────────────
def _place_start_position(self) -> None:
@@ -244,6 +273,8 @@ class PetWindow(QWidget):
if target is not None:
self._wander_target = target
self._commanded_move = True # overrides the idle-only rule
elif kind == "jump":
self.jump_to_monitor(int(action["monitor"]))
elif kind == "emote":
self.start_emote(action["emote"])
elif kind == "say":
@@ -378,6 +409,98 @@ class PetWindow(QWidget):
"""Stroll immediately (tray menu / anything that wants a nudge)."""
self._next_wander_at = 0.0
# ── monitors ─────────────────────────────────────────────────────────
def _build_monitors(self) -> list[Monitor]:
"""Snapshot Qt's screen list as plain dataclasses.
Full `geometry()`, not `availableGeometry()`: these coordinates are
what a screen grab gets cropped to, and a grab doesn't stop at the
taskbar. Placement uses availableGeometry separately.
"""
primary = QApplication.primaryScreen()
out = []
for index, screen in enumerate(QApplication.screens()):
geo = screen.geometry()
out.append(
Monitor(
index=index,
name=screen.name() or f"screen-{index + 1}",
x=geo.x(),
y=geo.y(),
width=geo.width(),
height=geo.height(),
primary=screen is primary,
)
)
return out
def publish_monitors(self, force: bool = True) -> None:
"""Push the current layout to whoever's listening (the controller).
*force* re-emits even when nothing changed, which is what the initial
wiring in ui/app.py needs: this window is built before the controller
exists, so the constructor's first publish goes to nobody.
"""
monitors = self._build_monitors()
changed = monitors != self._monitors
self._monitors = monitors
if changed or force:
self.monitors_changed.emit(monitors)
self._publish_pet_monitor(force=True)
def monitors(self) -> list[Monitor]:
return list(self._monitors)
def current_monitor_index(self) -> Optional[int]:
center = self.frameGeometry().center()
screens = QApplication.screens()
if not screens:
return None
screen = QApplication.screenAt(center)
if screen is not None:
try:
return screens.index(screen)
except ValueError:
pass
# Straddling a gap or dragged off the desktop entirely — fall back to
# whichever screen centre is nearest rather than reporting nothing.
best = min(
range(len(screens)),
key=lambda i: (screens[i].geometry().center() - center).manhattanLength(),
)
return best
def _publish_pet_monitor(self, force: bool = False) -> None:
index = self.current_monitor_index()
if index is None:
return
if force or index != self._pet_monitor:
self._pet_monitor = index
self.pet_monitor_changed.emit(index)
def jump_to_monitor(self, index: int) -> None:
"""Teleport to *index* (0-based, resolved by the controller) and land
with a hop. Instant rather than a stroll — Bolt asked to *jump*, and
walking between screens would take the long way across the desktop."""
screens = QApplication.screens()
if not (0 <= index < len(screens)):
return
geo = screens[index].availableGeometry()
point = self._clamp_to_screen(
QPoint(
geo.left() + (geo.width() - self.width()) // 2,
geo.top() + (geo.height() - self.height()) // 2,
),
geo,
)
self._stop_walking() # drop any stroll in flight, or it walks straight back
self.move(point)
self._schedule_next_wander()
self._publish_pet_monitor(force=True)
self.start_emote("hop")
self.update()
def _screen_geometry(self):
# screenAt() so a multi-monitor setup keeps the pet on the screen
# it's currently standing on rather than yanking it to the primary.
@@ -389,12 +512,17 @@ class PetWindow(QWidget):
self._next_wander_at = time.monotonic() + random.uniform(0.5 * base, 1.5 * base)
def _stop_walking(self) -> None:
if self._wander_target is None and not self._bob_offset:
if self._wander_target is None and not self._bob_offset and not self._walking:
return
self._wander_target = None
self._commanded_move = False
self._bob_phase = 0.0
self._bob_offset = 0
self._walking = False
self._walk_distance = 0.0
# Back to a standing frame, so the next stroll starts from a contact
# pose instead of mid-stride.
self.sprites.get(WALK).reset()
self.update()
def snap_to_edge(self) -> bool:
@@ -447,6 +575,13 @@ class PetWindow(QWidget):
wandering only happens when it's otherwise unoccupied."""
self._advance_emote()
self._wander_tick()
# Report crossing a screen boundary — by strolling, by being dragged,
# by anything. Guarded on the position actually changing so the common
# case (a stationary pet, 30x a second) costs one comparison.
position = self.pos()
if position != self._last_published_pos:
self._last_published_pos = position
self._publish_pet_monitor()
def _wander_tick(self) -> None:
# Only stroll while genuinely idle: not mid-drag, not napping, not
@@ -484,11 +619,51 @@ class PetWindow(QWidget):
self.snap_to_edge()
else:
self.move(round(here.x() + dx / distance * step), round(here.y() + dy / distance * step))
self._bob_phase += 0.45 # little walk-cycle hop
self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase))))
self._advance_walk(dx, dy, step)
self.update()
self._reposition_bubble()
def _advance_walk(self, dx: float, dy: float, step: float) -> None:
"""Drive the walk cycle from distance travelled (see the constant).
Falls back to the old bob-in-code if there's no walk art, so a sprite
folder without a walk/ directory still looks like it's moving rather
than sliding perfectly flat.
"""
self._walking = True
# Only turn on meaningful horizontal travel: a near-vertical stroll
# would otherwise flip him back and forth on rounding noise.
if abs(dx) > 1.0:
self._facing = 1 if dx > 0 else -1
if not self.sprites.has(WALK):
self._bob_phase += 0.45
self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase))))
return
self._bob_offset = 0 # the walk frames carry their own weight shift
self._walk_distance += step
while self._walk_distance >= _WALK_PIXELS_PER_FRAME:
self._walk_distance -= _WALK_PIXELS_PER_FRAME
self.sprites.get(WALK).advance()
def _animation_key(self):
"""Walking overrides the state animation — but only while genuinely
idle-and-moving, so he doesn't trot on the spot mid-sentence."""
if self._walking and self.sprites.has(WALK):
return WALK
return self._current_state
def _oriented(self, pixmap: Optional[QPixmap]) -> Optional[QPixmap]:
"""Mirror the (right-facing) walk art when he's heading left. Cached
per source frame — flipping on every paint would be wasteful at 30fps."""
if pixmap is None or self._facing >= 0:
return pixmap
key = pixmap.cacheKey()
mirrored = self._mirror_cache.get(key)
if mirrored is None:
mirrored = pixmap.transformed(QTransform().scale(-1, 1), Qt.SmoothTransformation)
self._mirror_cache[key] = mirrored
return mirrored
# ── state / speech ──────────────────────────────────────────────────
def set_state(self, state: PetState) -> None:
@@ -514,6 +689,11 @@ class PetWindow(QWidget):
# ── animation ────────────────────────────────────────────────────────
def _advance_frame(self) -> None:
# While walking the cycle is stepped by _advance_walk from distance
# travelled; letting this timer also advance it would double-step it
# and put the feet out of sync with the movement.
if self._walking and self.sprites.has(WALK):
return
self.sprites.get(self._current_state).advance()
self.update()
@@ -521,7 +701,10 @@ class PetWindow(QWidget):
painter = QPainter(self)
painter.setRenderHint(QPainter.Antialiasing)
painter.setRenderHint(QPainter.SmoothPixmapTransform)
pixmap: Optional[QPixmap] = self.sprites.get(self._current_state).current()
key = self._animation_key()
pixmap: Optional[QPixmap] = self.sprites.get(key).current()
if key == WALK:
pixmap = self._oriented(pixmap)
if pixmap is None:
self._apply_input_mask(None, 0, 0)
return
bolt_pet/ui/sprite.py
+35 -6
View File
@@ -95,17 +95,46 @@ def _load_frames_from_dir(directory: Path, size: int) -> list[QPixmap]:
return frames
WALK = "walk"
# Animations that aren't pipeline states. Walking is a property of *movement*,
# orthogonal to whether the pet is idle/listening/talking, so it deliberately
# isn't a PetState — state.py stays a description of the conversation, not of
# the body. Loaded the same way, keyed by name.
EXTRA_ANIMATIONS = (WALK,)
class SpriteSet:
"""All animations for every PetState, loaded from *sprite_dir*."""
"""All animations for every PetState, plus the extras, from *sprite_dir*."""
def __init__(self, sprite_dir: Path = DEFAULT_SPRITE_DIR, size: int = 160):
self.size = size
self._animations: dict[PetState, SpriteAnimation] = {}
self._animations: dict[str, SpriteAnimation] = {}
self._loaded: set[str] = set() # keys backed by real art, not placeholders
for state in PetState:
frames = _load_frames_from_dir(sprite_dir / state.value, size)
if not frames:
if frames:
self._loaded.add(state.value)
else:
frames = _placeholder_frames(state, size)
self._animations[state] = SpriteAnimation(frames)
self._animations[state.value] = SpriteAnimation(frames)
for name in EXTRA_ANIMATIONS:
frames = _load_frames_from_dir(sprite_dir / name, size)
if frames:
self._loaded.add(name)
self._animations[name] = SpriteAnimation(frames)
def get(self, state: PetState) -> SpriteAnimation:
return self._animations[state]
@staticmethod
def _key(key) -> str:
return key.value if isinstance(key, PetState) else str(key)
def get(self, key) -> SpriteAnimation:
"""Animation for a PetState or an extra name. Unknown/absent extras
fall back to idle, so a sprite folder with no walk/ still runs."""
return self._animations.get(self._key(key)) or self._animations[PetState.IDLE.value]
def has(self, key) -> bool:
"""True only when real frames were found — the caller uses this to
decide whether to use an extra animation at all, rather than being
handed a placeholder blob that looks nothing like walking."""
return self._key(key) in self._loaded