fix(computer): emit one full WHEEL_DELTA per libnut scroll tick (#2575)

* fix(computer): anchor untargeted libnut scrolls at viewport center

* style(computer): format scroll target helper

* fix(computer): focus active window before untargeted Windows scroll

* test(report): stabilize theme toggle e2e checks

* test(report): stabilize detail panel e2e checks

* test(report): restore Midscene e2e dogfooding

* test(report): simplify dogfood e2e prompts

* test(report): align detail panel dogfood assertion

Author: quanru

apps/report/e2e/detail-panel.yaml

@@ -19,15 +19,20 @@ tasks:
 
   - name: inspect detail side sections
     flow:
-      - ai: Click on the first task row in the left sidebar that has type "Planning" or "Insight" or "Locate"
+      - ai: Click on the first Planning task row in the left sidebar
       - sleep: 1000
-      - aiAssert: The Param section is visible in the right detail side panel and shows task parameters or input data
-      # Locate-type tasks render this section as "Element"; every other task
-      # renders it as "Output". Accept either label so the test is stable
-      # regardless of which task the previous step picked.
-      - ai: Click on the "Output" or "Element" section header in the right detail side panel to collapse that section
+      - aiAssert: The right detail side panel shows collapsible section headers named Param and Output, and the Param section is expanded with task input data
+      # Collapse both Param and the Output/Element section so the Meta section
+      # (always rendered last) rises to the top of the panel with its heading
+      # and first metadata rows fully on screen. This avoids depending on
+      # scroll position: Meta is long, so any partial scroll leaves either the
+      # heading or the type/status/start rows out of the viewport.
+      - aiTap: Output section header in the right detail side panel
+      - sleep: 500
+      - aiAssert: The Output section is collapsed; the Output header is still visible but the output content under it is hidden
+      - aiTap: Param section header in the right detail side panel
       - sleep: 1000
-      - aiAssert: The Meta section is visible in the right detail side panel and contains metadata entries such as type, status, start, end, or total time
+      - aiAssert: The Meta section is visible in the right detail side panel and shows metadata entries such as type, status, start, end, or total time; Param and Output are collapsed above it
 
   - name: open in playground button
     flow:

apps/report/e2e/report-single.yaml

@@ -32,18 +32,13 @@ tasks:
   # --- Dark mode ---
   - name: toggle dark mode
     flow:
-      - aiTap:
-        locate:
-          prompt: the "Toggle theme" button with a sun or moon icon in the top right header area
-          deepLocate: true
+      - aiTap: theme toggle button in the top right header
       - sleep: 2000
-      - aiAssert: The report UI (header, sidebar, and surrounding chrome) has a dark background color
+      - aiAssert: The report UI is in dark theme; the top header, left sidebar, and page chrome are dark charcoal or black rather than white
 
   - name: toggle back to light mode
     flow:
-      - aiTap:
-        locate:
-          prompt: the "Toggle theme" button with a sun or moon icon in the top right header area
-          deepLocate: true
+      - aiAssert: The report UI is currently in dark theme before clicking the theme toggle again
+      - aiTap: theme toggle button in the top right header
       - sleep: 2000
-      - aiAssert: The report UI (header, sidebar, and surrounding chrome) has a light/white background color
+      - aiAssert: The report UI is in light theme; the top header, left sidebar, and page chrome are light or white rather than dark charcoal or black

apps/report/e2e/theme-toggle.yaml

@@ -5,15 +5,16 @@ tasks:
   - name: toggle to dark mode
     flow:
       - sleep: 2000
-      - ai: Click the "Toggle theme" button in the top right header area
+      - aiTap: theme toggle button in the top right header
       - sleep: 2000
-      - aiAssert: The report UI (header, sidebar, and surrounding chrome) has a dark background color
+      - aiAssert: The report UI is in dark theme; the top header, left sidebar, and page chrome are dark charcoal or black rather than white
 
   - name: toggle back to light mode
     flow:
-      - ai: Click the "Toggle theme" button in the top right header area
+      - aiAssert: The report UI is currently in dark theme before clicking the theme toggle again
+      - aiTap: theme toggle button in the top right header
       - sleep: 2000
-      - aiAssert: The report UI (header, sidebar, and surrounding chrome) has a light/white background color
+      - aiAssert: The report UI is in light theme; the top header, left sidebar, and page chrome are light or white rather than dark charcoal or black
 
   - name: version info displayed
     flow:

packages/computer/src/device.ts

@@ -90,6 +90,35 @@ const EDGE_SCROLL_STEPS = 400;
 // minimum of 10 steps so small distances still feel momentum-like.
 const PHASED_PIXELS_PER_STEP = 30;
 const PHASED_MIN_STEPS = 10;
+// libnut fallback (Windows / Linux, and macOS when phased-scroll is
+// unavailable). libnut.scrollMouse(x, y) is *not* a portable pixel API:
+//   - macOS:   y is a CG pixel scroll delta
+//   - Linux:   y is the number of XButton4/5 press-release pairs (1 = one
+//              wheel notch)
+//   - Windows: y is forwarded directly to MOUSEEVENTF_WHEEL's `mouseData`,
+//              where the unit is WHEEL_DELTA (= 120) per detent
+// Calling scrollMouse(0, 6) on Windows therefore sends `mouseData = 6` —
+// far below one detent — which gets silently accumulated and is frequently
+// discarded by Chromium's WheelEventQueue (Electron apps like Lark/Feishu
+// see this as "scroll did nothing"). Always emit one detent per call, and
+// pace the calls so blink doesn't coalesce them into a single tick.
+const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
+const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
+const LIBNUT_FALLBACK_MAX_DETENTS = 200;
+const LIBNUT_FALLBACK_DETENT_AMOUNT = process.platform === 'win32' ? 120 : 1;
+// Edge scrolls (scrollToTop / scrollToBottom / ...) must drive all the way to
+// the boundary on every backend. The phased path requests EDGE_SCROLL_TOTAL_PX
+// (50_000 px); the libnut fallback aims for the same distance, capped at
+// LIBNUT_FALLBACK_MAX_DETENTS so a misconfigured screen size can't wedge the
+// process. Chromium clamps wheel events at the boundary, so overshooting is
+// free. Exported for regression coverage.
+export const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(
+  LIBNUT_FALLBACK_MAX_DETENTS,
+  Math.max(
+    1,
+    Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT),
+  ),
+);
 // Default scroll distance is 70% of the screen size on the relevant axis,
 // matching the web puppeteer/chrome-extension behavior so a model that simply
 // says "scroll down" without a distance gets a roughly one-screen scroll on
@@ -105,17 +134,20 @@ type EdgeScrollType =
 interface EdgeScrollStrategy {
   direction: ScrollDirection;
   key: 'home' | 'end';
+  // Unit vector for libnut.scrollMouse direction. Magnitude is applied
+  // separately by emitDetents() so each call is one full detent on every
+  // platform (see LIBNUT_FALLBACK_DETENT_AMOUNT).
   libnut: readonly [number, number];
 }
 
 // Single source of truth for edge scroll dispatch. Adding a new scrollType
 // only requires one entry here; the three backends (phased binary /
 // AppleScript / libnut) all read from the same spec.
 const EDGE_SCROLL_SPEC: Record<EdgeScrollType, EdgeScrollStrategy> = {
-  scrollToTop: { direction: 'up', key: 'home', libnut: [0, 10] },
-  scrollToBottom: { direction: 'down', key: 'end', libnut: [0, -10] },
-  scrollToLeft: { direction: 'left', key: 'home', libnut: [-10, 0] },
-  scrollToRight: { direction: 'right', key: 'end', libnut: [10, 0] },
+  scrollToTop: { direction: 'up', key: 'home', libnut: [0, 1] },
+  scrollToBottom: { direction: 'down', key: 'end', libnut: [0, -1] },
+  scrollToLeft: { direction: 'left', key: 'home', libnut: [-1, 0] },
+  scrollToRight: { direction: 'right', key: 'end', libnut: [1, 0] },
 };
 
 // macOS AppleScript key code mapping
@@ -1170,13 +1202,47 @@ Original error: ${lastRawMessage}`,
     this.inputDriver.sendKey(key, modifiers);
   }
 
-  private async performScroll(param: any): Promise<void> {
+  private resolveUntargetedScrollPoint(screenSize: Size): Point {
+    if (process.platform === 'win32') {
+      const activeWindowRect = this.inputDriver.getActiveWindowRect();
+      if (activeWindowRect) {
+        return {
+          x: activeWindowRect.x + activeWindowRect.width / 2,
+          y: activeWindowRect.y + activeWindowRect.height / 2,
+        };
+      }
+    }
+
+    return this.toGlobalPoint({
+      x: screenSize.width / 2,
+      y: screenSize.height / 2,
+    });
+  }
+
+  private async moveMouseToScrollTarget(param: any): Promise<Size | undefined> {
     if (param.locate) {
       const element = param.locate as LocateResultElement;
       const [x, y] = element.center;
       const point = this.toGlobalPoint({ x, y });
       this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
+      return undefined;
+    }
+
+    // Wheel events are delivered to the window under the cursor. For an
+    // untargeted "scroll down", anchor the cursor in the active viewport
+    // instead of relying on wherever the previous action left it.
+    const screenSize = await this.size();
+    if (process.platform === 'win32' && this.inputDriver.focusActiveWindow()) {
+      await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
     }
+    const point = this.resolveUntargetedScrollPoint(screenSize);
+    this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
+    await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
+    return screenSize;
+  }
+
+  private async performScroll(param: any): Promise<void> {
+    let screenSize = await this.moveMouseToScrollTarget(param);
 
     const scrollType = param?.scrollType;
 
@@ -1202,11 +1268,13 @@ Original error: ${lastRawMessage}`,
         return;
       }
 
-      const [dx, dy] = edgeSpec.libnut;
-      for (let i = 0; i < SCROLL_REPEAT_COUNT; i++) {
-        this.inputDriver.scrollMouse(dx, dy);
-        await this.inputDriver.delay(SCROLL_STEP_DELAY);
-      }
+      const [ux, uy] = edgeSpec.libnut;
+      await this.inputDriver.emitScrollDetents(
+        ux * LIBNUT_FALLBACK_DETENT_AMOUNT,
+        uy * LIBNUT_FALLBACK_DETENT_AMOUNT,
+        LIBNUT_FALLBACK_EDGE_DETENTS,
+        LIBNUT_FALLBACK_TICK_DELAY_MS,
+      );
       return;
     }
 
@@ -1220,9 +1288,8 @@ Original error: ${lastRawMessage}`,
       const isHorizontal = direction === 'left' || direction === 'right';
 
       let distance: number | undefined = param?.distance ?? undefined;
-      let screenSize: Size | undefined;
       if (!distance) {
-        screenSize = await this.size();
+        screenSize ??= await this.size();
         const base = isHorizontal ? screenSize.width : screenSize.height;
         distance = Math.max(
           1,
@@ -1253,16 +1320,23 @@ Original error: ${lastRawMessage}`,
         return;
       }
 
-      const ticks = Math.ceil(distance / 100);
-      const directionMap: Record<string, [number, number]> = {
-        up: [0, ticks],
-        down: [0, -ticks],
-        left: [-ticks, 0],
-        right: [ticks, 0],
+      const detents = Math.min(
+        LIBNUT_FALLBACK_MAX_DETENTS,
+        Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)),
+      );
+      const directionUnit: Record<string, [number, number]> = {
+        up: [0, 1],
+        down: [0, -1],
+        left: [-1, 0],
+        right: [1, 0],
       };
-
-      const [dx, dy] = directionMap[direction] || [0, -ticks];
-      this.inputDriver.scrollMouse(dx, dy);
+      const [ux, uy] = directionUnit[direction] || [0, -1];
+      await this.inputDriver.emitScrollDetents(
+        ux * LIBNUT_FALLBACK_DETENT_AMOUNT,
+        uy * LIBNUT_FALLBACK_DETENT_AMOUNT,
+        detents,
+        LIBNUT_FALLBACK_TICK_DELAY_MS,
+      );
       await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
       return;
     }

packages/computer/src/input-driver.ts

@@ -10,11 +10,21 @@ export interface LibNut {
   scrollMouse(x: number, y: number): void;
   keyTap(key: string, modifiers?: string[]): void;
   typeString(text: string): void;
+  getActiveWindow?(): number;
+  getWindowRect?(handle: number): WindowRect;
+  focusWindow?(handle: number): void;
 }
 
 export type MouseButton = 'left' | 'right' | 'middle';
 export type ScrollDirection = 'up' | 'down' | 'left' | 'right';
 
+export interface WindowRect {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
 interface ComputerInputDriverOptions {
   getLibnut(): LibNut | null;
   useAppleScript(): boolean;
@@ -56,6 +66,58 @@ export class ComputerInputDriver {
     this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
   }
 
+  focusActiveWindow(): boolean {
+    const lib = this.getLibnutOrThrow('focusActiveWindow');
+    if (
+      typeof lib.getActiveWindow !== 'function' ||
+      typeof lib.focusWindow !== 'function'
+    ) {
+      return false;
+    }
+
+    try {
+      const handle = lib.getActiveWindow();
+      if (!handle) return false;
+      lib.focusWindow(handle);
+      return true;
+    } catch (error) {
+      this.options.debug(`focusActiveWindow failed: ${error}`);
+      return false;
+    }
+  }
+
+  getActiveWindowRect(): WindowRect | null {
+    const lib = this.getLibnutOrThrow('getActiveWindowRect');
+    if (
+      typeof lib.getActiveWindow !== 'function' ||
+      typeof lib.getWindowRect !== 'function'
+    ) {
+      return null;
+    }
+
+    try {
+      const handle = lib.getActiveWindow();
+      if (!handle) return null;
+
+      const rect = lib.getWindowRect(handle);
+      if (
+        !Number.isFinite(rect.x) ||
+        !Number.isFinite(rect.y) ||
+        !Number.isFinite(rect.width) ||
+        !Number.isFinite(rect.height) ||
+        rect.width <= 0 ||
+        rect.height <= 0
+      ) {
+        return null;
+      }
+
+      return rect;
+    } catch (error) {
+      this.options.debug(`getActiveWindowRect failed: ${error}`);
+      return null;
+    }
+  }
+
   mouseClick(button?: MouseButton, double?: boolean): void {
     const lib = this.getLibnutOrThrow('mouseClick');
     // libnut is a native binding that distinguishes "no argument" from
@@ -78,6 +140,29 @@ export class ComputerInputDriver {
     this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
   }
 
+  /**
+   * Emit one `libnut.scrollMouse` call per detent and pace them with
+   * `delayMs`. Per-call magnitude is fixed by the caller so each call is
+   * exactly one detent on the target platform — on Windows the libnut
+   * binding forwards `mouseData` straight to `MOUSEEVENTF_WHEEL`, where
+   * sub-WHEEL_DELTA values (< 120) get accumulated and frequently dropped
+   * by Chromium's WheelEventQueue.
+   */
+  async emitScrollDetents(
+    dx: number,
+    dy: number,
+    detents: number,
+    delayMs: number,
+  ): Promise<void> {
+    this.assertActive('emitScrollDetents');
+    for (let i = 0; i < detents; i++) {
+      this.scrollMouse(dx, dy);
+      if (i < detents - 1) {
+        await this.delay(delayMs);
+      }
+    }
+  }
+
   keyTap(key: string, modifiers?: string[]): void {
     const lib = this.getLibnutOrThrow('keyTap');
     // See note on mouseClick — avoid passing explicit undefined to libnut.