Multi-monitor jumps, screen OCR, and generated sprite art
petctl gains screen verbs: `jump` (1-based number, name, next/prev/ primary/other, or a direction resolved from real geometry), `monitors`, and `read` for OCR of a monitor's contents. - monitors.py: pure layout model + jump-target resolution. The monitor list is published by PetWindow from QGuiApplication.screens() over a queued signal, so the controller and window agree on what "monitor 2" means; xrandr and Qt order screens differently on the same machine. - screen_text.py: pull-only OCR (mss capture + Tesseract/RapidOCR). Nothing captures unless the server asks, and the text rides back up the tool-result relay so Bolt can read a screen mid-turn. Both deps optional, soft-failing with a reason. SCREEN_TEXT=false removes it. - Query verbs are answered in controller._handle_command rather than pet_actions.describe(), because their output is the point. - scripts/generate_bolt_sprites.py draws every frame; walk/ is a side-view cycle stepped by distance travelled, not by the animation timer, so the planted paw tracks the window exactly. sprite.py loads it via EXTRA_ANIMATIONS keyed by name, with has() so callers can decline a placeholder blob. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+188
-5
@@ -20,8 +20,9 @@ from PySide6.QtGui import (
|
||||
from PySide6.QtWidgets import QApplication, QWidget
|
||||
|
||||
from .. import config
|
||||
from ..monitors import Monitor
|
||||
from ..state import PetState
|
||||
from .sprite import SpriteSet
|
||||
from .sprite import WALK, SpriteSet
|
||||
|
||||
_DRAG_THRESHOLD_PX = 4
|
||||
# Movement runs on its own ~30fps timer, independent of the (slower) sprite
|
||||
@@ -29,6 +30,12 @@ _DRAG_THRESHOLD_PX = 4
|
||||
_WANDER_TICK_MS = 33
|
||||
_EMOTE_TICKS = 36 # ~1.2s per emote at the tick rate above
|
||||
_NAP_OPACITY = 0.35
|
||||
# How far the pet travels per walk-cycle frame. The cycle is advanced by
|
||||
# distance rather than by the animation clock so a planted paw tracks backwards
|
||||
# at exactly the speed the window moves forwards — drive it off a timer instead
|
||||
# and the feet skate whenever PET_WANDER_SPEED doesn't happen to match the fps.
|
||||
# Eight frames at 13px is a ~104px stride cycle, a bit under the pet's width.
|
||||
_WALK_PIXELS_PER_FRAME = 13.0
|
||||
|
||||
|
||||
def emote_transform(emote: str, progress: float) -> tuple[float, float, float, float]:
|
||||
@@ -164,6 +171,12 @@ class SpeechBubble(QWidget):
|
||||
class PetWindow(QWidget):
|
||||
talk_requested = Signal()
|
||||
copied = Signal(str) # bubble text the user just put on the clipboard
|
||||
# The screen layout, published *to* the controller (queued, cross-thread).
|
||||
# The window is the only thing allowed to ask Qt about screens, so the
|
||||
# controller and the window can never disagree about what "monitor 2"
|
||||
# means — see monitors.py.
|
||||
monitors_changed = Signal(list) # list[monitors.Monitor]
|
||||
pet_monitor_changed = Signal(int) # 0-based index the pet is standing on
|
||||
|
||||
def __init__(self, sprite_dir: Optional[Path] = None, size: Optional[int] = None):
|
||||
super().__init__()
|
||||
@@ -201,6 +214,10 @@ class PetWindow(QWidget):
|
||||
self._next_wander_at = 0.0
|
||||
self._bob_offset = 0
|
||||
self._bob_phase = 0.0
|
||||
self._walking = False
|
||||
self._facing = 1 # +1 right, -1 left; the walk art is drawn facing right
|
||||
self._walk_distance = 0.0
|
||||
self._mirror_cache: dict[int, QPixmap] = {}
|
||||
self._schedule_next_wander()
|
||||
self._wander_timer = QTimer(self)
|
||||
self._wander_timer.timeout.connect(self._movement_tick)
|
||||
@@ -210,6 +227,18 @@ class PetWindow(QWidget):
|
||||
self.set_click_through(config.PET_CLICK_THROUGH)
|
||||
self._place_start_position()
|
||||
|
||||
self._monitors: list[Monitor] = []
|
||||
self._pet_monitor: Optional[int] = None
|
||||
self._last_published_pos: Optional[QPoint] = None
|
||||
app = QApplication.instance()
|
||||
if app is not None:
|
||||
# Screens come and go — a laptop docking, a TV waking up. Republish
|
||||
# rather than letting Bolt jump to a monitor that's been unplugged.
|
||||
app.screenAdded.connect(lambda _s: self.publish_monitors())
|
||||
app.screenRemoved.connect(lambda _s: self.publish_monitors())
|
||||
app.primaryScreenChanged.connect(lambda _s: self.publish_monitors())
|
||||
self.publish_monitors()
|
||||
|
||||
# ── placement ────────────────────────────────────────────────────────
|
||||
|
||||
def _place_start_position(self) -> None:
|
||||
@@ -244,6 +273,8 @@ class PetWindow(QWidget):
|
||||
if target is not None:
|
||||
self._wander_target = target
|
||||
self._commanded_move = True # overrides the idle-only rule
|
||||
elif kind == "jump":
|
||||
self.jump_to_monitor(int(action["monitor"]))
|
||||
elif kind == "emote":
|
||||
self.start_emote(action["emote"])
|
||||
elif kind == "say":
|
||||
@@ -378,6 +409,98 @@ class PetWindow(QWidget):
|
||||
"""Stroll immediately (tray menu / anything that wants a nudge)."""
|
||||
self._next_wander_at = 0.0
|
||||
|
||||
# ── monitors ─────────────────────────────────────────────────────────
|
||||
|
||||
def _build_monitors(self) -> list[Monitor]:
|
||||
"""Snapshot Qt's screen list as plain dataclasses.
|
||||
|
||||
Full `geometry()`, not `availableGeometry()`: these coordinates are
|
||||
what a screen grab gets cropped to, and a grab doesn't stop at the
|
||||
taskbar. Placement uses availableGeometry separately.
|
||||
"""
|
||||
primary = QApplication.primaryScreen()
|
||||
out = []
|
||||
for index, screen in enumerate(QApplication.screens()):
|
||||
geo = screen.geometry()
|
||||
out.append(
|
||||
Monitor(
|
||||
index=index,
|
||||
name=screen.name() or f"screen-{index + 1}",
|
||||
x=geo.x(),
|
||||
y=geo.y(),
|
||||
width=geo.width(),
|
||||
height=geo.height(),
|
||||
primary=screen is primary,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
def publish_monitors(self, force: bool = True) -> None:
|
||||
"""Push the current layout to whoever's listening (the controller).
|
||||
|
||||
*force* re-emits even when nothing changed, which is what the initial
|
||||
wiring in ui/app.py needs: this window is built before the controller
|
||||
exists, so the constructor's first publish goes to nobody.
|
||||
"""
|
||||
monitors = self._build_monitors()
|
||||
changed = monitors != self._monitors
|
||||
self._monitors = monitors
|
||||
if changed or force:
|
||||
self.monitors_changed.emit(monitors)
|
||||
self._publish_pet_monitor(force=True)
|
||||
|
||||
def monitors(self) -> list[Monitor]:
|
||||
return list(self._monitors)
|
||||
|
||||
def current_monitor_index(self) -> Optional[int]:
|
||||
center = self.frameGeometry().center()
|
||||
screens = QApplication.screens()
|
||||
if not screens:
|
||||
return None
|
||||
screen = QApplication.screenAt(center)
|
||||
if screen is not None:
|
||||
try:
|
||||
return screens.index(screen)
|
||||
except ValueError:
|
||||
pass
|
||||
# Straddling a gap or dragged off the desktop entirely — fall back to
|
||||
# whichever screen centre is nearest rather than reporting nothing.
|
||||
best = min(
|
||||
range(len(screens)),
|
||||
key=lambda i: (screens[i].geometry().center() - center).manhattanLength(),
|
||||
)
|
||||
return best
|
||||
|
||||
def _publish_pet_monitor(self, force: bool = False) -> None:
|
||||
index = self.current_monitor_index()
|
||||
if index is None:
|
||||
return
|
||||
if force or index != self._pet_monitor:
|
||||
self._pet_monitor = index
|
||||
self.pet_monitor_changed.emit(index)
|
||||
|
||||
def jump_to_monitor(self, index: int) -> None:
|
||||
"""Teleport to *index* (0-based, resolved by the controller) and land
|
||||
with a hop. Instant rather than a stroll — Bolt asked to *jump*, and
|
||||
walking between screens would take the long way across the desktop."""
|
||||
screens = QApplication.screens()
|
||||
if not (0 <= index < len(screens)):
|
||||
return
|
||||
geo = screens[index].availableGeometry()
|
||||
point = self._clamp_to_screen(
|
||||
QPoint(
|
||||
geo.left() + (geo.width() - self.width()) // 2,
|
||||
geo.top() + (geo.height() - self.height()) // 2,
|
||||
),
|
||||
geo,
|
||||
)
|
||||
self._stop_walking() # drop any stroll in flight, or it walks straight back
|
||||
self.move(point)
|
||||
self._schedule_next_wander()
|
||||
self._publish_pet_monitor(force=True)
|
||||
self.start_emote("hop")
|
||||
self.update()
|
||||
|
||||
def _screen_geometry(self):
|
||||
# screenAt() so a multi-monitor setup keeps the pet on the screen
|
||||
# it's currently standing on rather than yanking it to the primary.
|
||||
@@ -389,12 +512,17 @@ class PetWindow(QWidget):
|
||||
self._next_wander_at = time.monotonic() + random.uniform(0.5 * base, 1.5 * base)
|
||||
|
||||
def _stop_walking(self) -> None:
|
||||
if self._wander_target is None and not self._bob_offset:
|
||||
if self._wander_target is None and not self._bob_offset and not self._walking:
|
||||
return
|
||||
self._wander_target = None
|
||||
self._commanded_move = False
|
||||
self._bob_phase = 0.0
|
||||
self._bob_offset = 0
|
||||
self._walking = False
|
||||
self._walk_distance = 0.0
|
||||
# Back to a standing frame, so the next stroll starts from a contact
|
||||
# pose instead of mid-stride.
|
||||
self.sprites.get(WALK).reset()
|
||||
self.update()
|
||||
|
||||
def snap_to_edge(self) -> bool:
|
||||
@@ -447,6 +575,13 @@ class PetWindow(QWidget):
|
||||
wandering only happens when it's otherwise unoccupied."""
|
||||
self._advance_emote()
|
||||
self._wander_tick()
|
||||
# Report crossing a screen boundary — by strolling, by being dragged,
|
||||
# by anything. Guarded on the position actually changing so the common
|
||||
# case (a stationary pet, 30x a second) costs one comparison.
|
||||
position = self.pos()
|
||||
if position != self._last_published_pos:
|
||||
self._last_published_pos = position
|
||||
self._publish_pet_monitor()
|
||||
|
||||
def _wander_tick(self) -> None:
|
||||
# Only stroll while genuinely idle: not mid-drag, not napping, not
|
||||
@@ -484,11 +619,51 @@ class PetWindow(QWidget):
|
||||
self.snap_to_edge()
|
||||
else:
|
||||
self.move(round(here.x() + dx / distance * step), round(here.y() + dy / distance * step))
|
||||
self._bob_phase += 0.45 # little walk-cycle hop
|
||||
self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase))))
|
||||
self._advance_walk(dx, dy, step)
|
||||
self.update()
|
||||
self._reposition_bubble()
|
||||
|
||||
def _advance_walk(self, dx: float, dy: float, step: float) -> None:
|
||||
"""Drive the walk cycle from distance travelled (see the constant).
|
||||
|
||||
Falls back to the old bob-in-code if there's no walk art, so a sprite
|
||||
folder without a walk/ directory still looks like it's moving rather
|
||||
than sliding perfectly flat.
|
||||
"""
|
||||
self._walking = True
|
||||
# Only turn on meaningful horizontal travel: a near-vertical stroll
|
||||
# would otherwise flip him back and forth on rounding noise.
|
||||
if abs(dx) > 1.0:
|
||||
self._facing = 1 if dx > 0 else -1
|
||||
if not self.sprites.has(WALK):
|
||||
self._bob_phase += 0.45
|
||||
self._bob_offset = int(round(-2.5 * abs(math.sin(self._bob_phase))))
|
||||
return
|
||||
self._bob_offset = 0 # the walk frames carry their own weight shift
|
||||
self._walk_distance += step
|
||||
while self._walk_distance >= _WALK_PIXELS_PER_FRAME:
|
||||
self._walk_distance -= _WALK_PIXELS_PER_FRAME
|
||||
self.sprites.get(WALK).advance()
|
||||
|
||||
def _animation_key(self):
|
||||
"""Walking overrides the state animation — but only while genuinely
|
||||
idle-and-moving, so he doesn't trot on the spot mid-sentence."""
|
||||
if self._walking and self.sprites.has(WALK):
|
||||
return WALK
|
||||
return self._current_state
|
||||
|
||||
def _oriented(self, pixmap: Optional[QPixmap]) -> Optional[QPixmap]:
|
||||
"""Mirror the (right-facing) walk art when he's heading left. Cached
|
||||
per source frame — flipping on every paint would be wasteful at 30fps."""
|
||||
if pixmap is None or self._facing >= 0:
|
||||
return pixmap
|
||||
key = pixmap.cacheKey()
|
||||
mirrored = self._mirror_cache.get(key)
|
||||
if mirrored is None:
|
||||
mirrored = pixmap.transformed(QTransform().scale(-1, 1), Qt.SmoothTransformation)
|
||||
self._mirror_cache[key] = mirrored
|
||||
return mirrored
|
||||
|
||||
# ── state / speech ──────────────────────────────────────────────────
|
||||
|
||||
def set_state(self, state: PetState) -> None:
|
||||
@@ -514,6 +689,11 @@ class PetWindow(QWidget):
|
||||
# ── animation ────────────────────────────────────────────────────────
|
||||
|
||||
def _advance_frame(self) -> None:
|
||||
# While walking the cycle is stepped by _advance_walk from distance
|
||||
# travelled; letting this timer also advance it would double-step it
|
||||
# and put the feet out of sync with the movement.
|
||||
if self._walking and self.sprites.has(WALK):
|
||||
return
|
||||
self.sprites.get(self._current_state).advance()
|
||||
self.update()
|
||||
|
||||
@@ -521,7 +701,10 @@ class PetWindow(QWidget):
|
||||
painter = QPainter(self)
|
||||
painter.setRenderHint(QPainter.Antialiasing)
|
||||
painter.setRenderHint(QPainter.SmoothPixmapTransform)
|
||||
pixmap: Optional[QPixmap] = self.sprites.get(self._current_state).current()
|
||||
key = self._animation_key()
|
||||
pixmap: Optional[QPixmap] = self.sprites.get(key).current()
|
||||
if key == WALK:
|
||||
pixmap = self._oriented(pixmap)
|
||||
if pixmap is None:
|
||||
self._apply_input_mask(None, 0, 0)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user