fix: make directional gesture presets orientation-aware

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- iOS directional gesture presets (`gesture scroll-up/down/left/right`, `swipe-from-*-edge`) are now orientation-aware (#66). They previously emitted device-native portrait axes, so on a rotated device `scroll-up` pointed 90° (landscape) or 180° (upside-down) away from its name — no scroll, or an unintended row navigation. Preset math now runs in the current visual space (auto-calibrated per command, the same #34 mapping `tap` selectors use) and the endpoints are transformed into HID coordinates; this covers both the standalone command and `ios batch` gesture steps (which share the batch-wide calibration). When the orientation cannot be determined the gesture falls back to the legacy portrait dispatch and says so via the `advisory` envelope key. Explicit `swipe`/`touch` coordinates remain device-native portrait by contract; pinch/rotate presets are likewise unchanged.
+
+### Changed
+
+- `gesture --screen-width/--screen-height` on iOS: for **single-finger presets** they now describe a visual-space canvas and default to the auto-detected screen size of the current orientation (previously always 390×844, iPhone 15 portrait — wrong for iPads and every landscape state); 390×844 survives only when the screen size cannot be probed. **Pinch/rotate presets** keep the raw device-native portrait canvas with the fixed 390×844 default.
+
 ### Added
 
 - The agent-eval suite can now pin exactly which sim-use binary a run evaluates: `make eval ARGS="-b <path>"` / `scripts/eval.sh --sim-use <path>` / `run.py --sim-use <path>` (default remains whatever `sim-use` resolves to on PATH). The wrapper and runner print `sim-use under test: <real path> (<version>)` up front and the report header records it, so a run can never silently exercise the wrong binary — and development builds under `.build/` can be evaluated directly. New repo skill `.claude/skills/run-evals/` orchestrates the whole flow for agents and contributors: environment prep (Device Hub closed, fixtures installed), binary selection, cost confirmation, and verdict triage; `e2e/agent-evals/README.md` documents the new prereqs and flags.

README.md

@@ -208,6 +208,8 @@ sim-use gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --device $UDID
 
 `--pre-delay` / `--post-delay` / `--duration` work on `tap`, `swipe`, and `gesture` alike for coarse timing control.
 
+Single-finger presets (`scroll-*`, `swipe-from-*-edge`) name **visual** directions and are orientation-aware on iOS: their canvas size and rotation are auto-detected per command, so `scroll-up` scrolls the on-screen content up whether the device is portrait, landscape, or upside-down. Explicit `swipe`/`touch` coordinates remain device-native portrait space by contract; pinch/rotate presets are likewise untransformed.
+
 ### Text input
 
 ```bash

Sources/SimUse/Commands/Gesture.swift

@@ -37,9 +37,12 @@ struct Gesture: SimUseExecutableCommand {
           sim-use gesture rotate-cw --angle 45 --udid SIMULATOR_UDID
 
         Platforms:
-          * iOS — coordinates default to iPhone 15 (390×844); pass --screen-width
-            and --screen-height for other devices. HID granularity is controlled
-            by --delta (single-finger) or --steps / --step-ms (multi-touch).
+          * iOS — single-finger presets are orientation-aware: their math runs
+            in the current visual space (auto-detected size and rotation), so
+            scroll-up scrolls content up on a rotated device too. Pinch/rotate
+            presets remain device-native portrait space (390×844 default).
+            HID granularity is controlled by --delta (single-finger) or
+            --steps / --step-ms (multi-touch).
           * Android — coordinates default to the device's real display in pixels
             (auto-detected via the bridge). --delta, --steps, --step-ms are
             iOS-HID-specific and silently ignored on Android, since
@@ -50,10 +53,10 @@ struct Gesture: SimUseExecutableCommand {
     @Argument(help: "The gesture preset to perform.")
     var preset: GesturePreset
 
-    @Option(name: .customLong("screen-width"), help: "Screen width in points (default: 390 for iPhone 15).")
+    @Option(name: .customLong("screen-width"), help: "Canvas width for the preset math. iOS single-finger presets: visual space, auto-detected by default (390 fallback); iOS pinch/rotate presets: device-native portrait, fixed 390 default. Android: real display pixels, auto-detected for all presets.")
     var screenWidth: Double?
 
-    @Option(name: .customLong("screen-height"), help: "Screen height in points (default: 844 for iPhone 15).")
+    @Option(name: .customLong("screen-height"), help: "Canvas height for the preset math. iOS single-finger presets: visual space, auto-detected by default (844 fallback); iOS pinch/rotate presets: device-native portrait, fixed 844 default. Android: real display pixels, auto-detected for all presets.")
     var screenHeight: Double?
 
     @Option(name: .customLong("duration"), help: "Duration of the gesture in seconds. Defaults to the preset baseline (0.3s edge / 0.5s scroll+pinch+rotate), except rotate presets auto-extend to |angle|/180s for sweeps > 90° so angular velocity stays near 180°/sec (recogniser sweet spot). Pass explicitly to override.")

Sources/iOSSimBackend/A11y/AccessibilityFetcher.swift

@@ -248,6 +248,33 @@ public struct AccessibilityFetcher {
         return FetchResult(data: data, calibration: finalCalibration)
     }
 
+    /// Orientation calibration WITHOUT the collapsed-children recovery
+    /// walk — for verbs that need the current orientation but not the
+    /// tree itself (directional gesture presets, issue #66). Costs one
+    /// tree-fetch XPC plus at most `OrientationCalibrator.defaultMaxProbes`
+    /// hit-test probes; recovery's quadtree probing (up to hundreds of
+    /// XPC round-trips on WebView-heavy screens) is skipped entirely.
+    public static func fetchOrientationCalibration(
+        for simulatorUDID: String,
+        logger: SimUseLogger
+    ) async throws -> OrientationCalibration {
+        let simulatorSet = try await getSimulatorSet(
+            deviceSetPath: nil,
+            logger: logger,
+            reporter: EmptyEventReporter.shared
+        )
+        guard let target = simulatorSet.allSimulators.first(where: { $0.udid == simulatorUDID }) else {
+            throw CLIError(errorDescription: "Simulator with UDID \(simulatorUDID) not found in set.")
+        }
+        let native = NativePortraitSize(screenInfo: target.screenInfo)
+        let probe: CollapsedChildrenRecovery.PointProbe = { point in
+            let raw: AnyObject = try await target.legacyAccessibilityElement(at: point, nestedFormat: false)
+            return raw as? [String: Any]
+        }
+        let info: AnyObject = try await target.legacyAccessibilityElements(nestedFormat: true)
+        return await calibrate(info: info, native: native, probe: probe, logger: logger)
+    }
+
     // MARK: - Calibration over the raw tree payload
 
     private static func calibrate(

Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: Apache-2.0
+import Foundation
+import SimUseCore
+
+/// Pure decision layer for orientation-aware directional gestures
+/// (issue #66). Named presets (`scroll-*`, `swipe-from-*-edge`)
+/// describe VISUAL directions, so their stroke math must run in the
+/// UI space of the current interface orientation and the resulting
+/// endpoints must cross into the native-portrait framebuffer space
+/// the HID layer consumes — the same `uiToFramebuffer` mapping tap
+/// selectors ride (issue #34). Explicit user coordinates never pass
+/// through here.
+public enum GestureOrientationMapping {
+
+    /// The legacy preset canvas, kept as the last-resort fallback when
+    /// no native screen size is known (identity calibration).
+    public static let legacyWidth = 390.0
+    public static let legacyHeight = 844.0
+
+    /// The visual-space canvas the preset math runs in. Explicit
+    /// `--screen-width` / `--screen-height` flags win per axis (they
+    /// describe a visual-space viewport); missing axes come from the
+    /// calibrated UI size, or the legacy 390x844 when the native size
+    /// is unknown.
+    public static func visualSize(
+        explicitWidth: Double?,
+        explicitHeight: Double?,
+        calibration: OrientationCalibration
+    ) -> (width: Double, height: Double) {
+        let ui = calibration.native.map { calibration.orientation.uiSize(native: $0) }
+        return (
+            explicitWidth ?? ui?.width ?? legacyWidth,
+            explicitHeight ?? ui?.height ?? legacyHeight
+        )
+    }
+
+    /// Maps one UI-space stroke's endpoints into HID (native-portrait
+    /// framebuffer) coordinates. Identity (portrait or no native size)
+    /// passes the stroke through bit-for-bit.
+    public static func hidStroke(
+        _ stroke: GesturePreset.Stroke,
+        calibration: OrientationCalibration
+    ) -> (startX: Double, startY: Double, endX: Double, endY: Double) {
+        let start = calibration.hidCGPoint(CGPoint(x: stroke.startX, y: stroke.startY))
+        let end = calibration.hidCGPoint(CGPoint(x: stroke.endX, y: stroke.endY))
+        return (Double(start.x), Double(start.y), Double(end.x), Double(end.y))
+    }
+}

Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift

@@ -123,12 +123,38 @@ extension IOSSimSwipeCommand: BatchConvertible {
 
 extension IOSSimGestureCommand: BatchConvertible {
     public func toBatchPrimitives(context: BatchContext, logger: SimUseLogger) async throws -> [BatchPrimitive] {
-        let width = screenWidth ?? 390.0
-        let height = screenHeight ?? 844.0
-        let coords = preset.coordinates(screenWidth: width, screenHeight: height)
         let gestureDuration = duration ?? preset.defaultDuration
         let gestureDelta = delta ?? preset.defaultDelta
 
+        let coords: (startX: Double, startY: Double, endX: Double, endY: Double)
+        if preset.isMultiTouch {
+            // Multi-touch presets keep the legacy raw dispatch (see the
+            // standalone path for the scope rationale).
+            let width = screenWidth ?? GestureOrientationMapping.legacyWidth
+            let height = screenHeight ?? GestureOrientationMapping.legacyHeight
+            coords = preset.coordinates(screenWidth: width, screenHeight: height)
+        } else {
+            // Directional presets ride the batch-wide calibration
+            // (issue #66) — computed lazily once per run and shared
+            // with tap selector steps. An unreachable AX tree degrades
+            // to an identity dispatch with a recorded advisory instead
+            // of failing the whole batch.
+            let calibration: OrientationCalibration
+            if let roots = try? await context.accessibilityRoots(logger: logger) {
+                calibration = await context.orientationCalibration(roots: roots, logger: logger)
+            } else {
+                calibration = .identity()
+                context.recordAdvisory(CommandAdvisory(
+                    kind: .orientationCalibrationFallback,
+                    message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated."
+                ))
+            }
+            let visual = GestureOrientationMapping.visualSize(
+                explicitWidth: screenWidth, explicitHeight: screenHeight, calibration: calibration)
+            let stroke = preset.strokes(screenWidth: visual.width, screenHeight: visual.height)[0]
+            coords = GestureOrientationMapping.hidStroke(stroke, calibration: calibration)
+        }
+
         let gestureEvent = FBSimulatorHIDEvent.swipe(
             coords.startX,
             yStart: coords.startY,

Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift

@@ -10,8 +10,18 @@ import SimUseCore
 /// `sim-use ios gesture`. The top-level command resolves the target
 /// platform via `PlatformRouter` and forwards iOS UDIDs through here.
 public struct IOSSimGestureCommand: SimUseExecutableCommand {
-    public struct ExecutionResult: Codable {
-        public init() {}
+    public struct ExecutionResult: Codable, CommandAdvisoryProviding {
+        /// Excluded from the encoded `data` payload via `CodingKeys`
+        /// (the default value keeps decode synthesis working) — the
+        /// envelope hoists it to the top-level `advisory` key. See
+        /// `CommandAdvisoryProviding` for the contract.
+        public var commandAdvisory: CommandAdvisory? = nil
+
+        public init(commandAdvisory: CommandAdvisory? = nil) {
+            self.commandAdvisory = commandAdvisory
+        }
+
+        private enum CodingKeys: CodingKey {}
     }
 
     public static let configuration = CommandConfiguration(
@@ -20,7 +30,9 @@ public struct IOSSimGestureCommand: SimUseExecutableCommand {
         discussion: """
         Execute common gesture patterns without specifying coordinates.
 
-        Single-finger presets:
+        Single-finger presets (visual directions — orientation-aware,
+        so scroll-up scrolls the on-screen content up on a rotated
+        device too):
           scroll-up, scroll-down, scroll-left, scroll-right
           swipe-from-left-edge, swipe-from-right-edge
           swipe-from-top-edge, swipe-from-bottom-edge
@@ -34,10 +46,10 @@ public struct IOSSimGestureCommand: SimUseExecutableCommand {
     @Argument(help: "The gesture preset to perform.")
     public var preset: GesturePreset
 
-    @Option(name: .customLong("screen-width"), help: "Screen width in points (default: 390 for iPhone 15).")
+    @Option(name: .customLong("screen-width"), help: "Canvas width in points for the preset math. Single-finger presets: visual space, auto-detected from the current orientation by default (390 when the screen size is unknown). Pinch/rotate presets: device-native portrait space, fixed 390 default.")
     public var screenWidth: Double?
 
-    @Option(name: .customLong("screen-height"), help: "Screen height in points (default: 844 for iPhone 15).")
+    @Option(name: .customLong("screen-height"), help: "Canvas height in points for the preset math. Single-finger presets: visual space, auto-detected from the current orientation by default (844 when the screen size is unknown). Pinch/rotate presets: device-native portrait space, fixed 844 default.")
     public var screenHeight: Double?
 
     @Option(name: .customLong("duration"), help: "Duration of the gesture in seconds (uses preset default if not specified).")
@@ -144,42 +156,81 @@ public struct IOSSimGestureCommand: SimUseExecutableCommand {
         try await setup(logger: logger)
         try await performGlobalSetup(logger: logger)
 
-        let width = screenWidth ?? 390.0
-        let height = screenHeight ?? 844.0
         // `recommendedDuration` auto-extends rotate sweeps beyond 90°
         // to keep angular velocity near 180°/sec (recogniser sweet
         // spot). Pinch / scroll / edge presets fall through to the
         // baseline `defaultDuration` unchanged.
         let gestureDuration = duration ?? preset.recommendedDuration(angle: angle)
 
         logger.info().log("Performing \(preset.description)")
-        logger.info().log("Screen size: \(width)x\(height)")
         logger.info().log("Duration: \(gestureDuration)s")
 
         if preset.isMultiTouch {
+            // Multi-touch presets stay in device-native portrait axes:
+            // orientation-aware pinch/rotate would need every
+            // interpolated sample mapped, not just the endpoints, and
+            // is out of scope for the issue #66 fix.
+            let width = screenWidth ?? GestureOrientationMapping.legacyWidth
+            let height = screenHeight ?? GestureOrientationMapping.legacyHeight
+            logger.info().log("Screen size: \(width)x\(height)")
             try await runMultiTouch(
                 width: width,
                 height: height,
                 duration: gestureDuration,
                 logger: logger
             )
-        } else {
-            try await runSingleTouch(
-                width: width,
-                height: height,
-                duration: gestureDuration,
-                logger: logger
-            )
+            logger.info().log("Gesture completed successfully")
+            return ExecutionResult()
         }
 
+        // Directional presets name a visual direction, so their math
+        // runs in the calibrated UI space and the endpoints cross into
+        // HID space (issue #66) — the same mapping tap selectors use.
+        let calibration = await Self.orientationCalibration(
+            udid: device.resolved, preset: preset, logger: logger)
+        let visual = GestureOrientationMapping.visualSize(
+            explicitWidth: screenWidth, explicitHeight: screenHeight, calibration: calibration)
+        logger.info().log("Visual canvas: \(visual.width)x\(visual.height) (orientation: \(calibration.orientation.rawValue))")
+        try await runSingleTouch(
+            visual: visual,
+            calibration: calibration,
+            duration: gestureDuration,
+            logger: logger
+        )
+
         logger.info().log("Gesture completed successfully")
-        return ExecutionResult()
+        return ExecutionResult(commandAdvisory: calibration.advisory)
+    }
+
+    /// The current orientation for a directional preset, degrading to
+    /// an identity (portrait) dispatch with an explicit advisory when
+    /// the simulator cannot be calibrated at all — never silently.
+    static func orientationCalibration(
+        udid: String,
+        preset: GesturePreset,
+        logger: SimUseLogger
+    ) async -> OrientationCalibration {
+        do {
+            return try await AccessibilityFetcher.fetchOrientationCalibration(for: udid, logger: logger)
+        } catch {
+            logger.info().log("Orientation calibration unavailable (\(error.localizedDescription)); dispatching \(preset.rawValue) in native portrait axes")
+            return .identity(advisory: CommandAdvisory(
+                kind: .orientationCalibrationFallback,
+                message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated."
+            ))
+        }
     }
 
-    private func runSingleTouch(width: Double, height: Double, duration: Double, logger: SimUseLogger) async throws {
-        let coords = preset.coordinates(screenWidth: width, screenHeight: height)
+    private func runSingleTouch(
+        visual: (width: Double, height: Double),
+        calibration: OrientationCalibration,
+        duration: Double,
+        logger: SimUseLogger
+    ) async throws {
+        let stroke = preset.strokes(screenWidth: visual.width, screenHeight: visual.height)[0]
+        let coords = GestureOrientationMapping.hidStroke(stroke, calibration: calibration)
         let gestureDelta = delta ?? preset.defaultDelta
-        logger.info().log("Coordinates: (\(coords.startX), \(coords.startY)) to (\(coords.endX), \(coords.endY))")
+        logger.info().log("Coordinates (HID space): (\(coords.startX), \(coords.startY)) to (\(coords.endX), \(coords.endY))")
         logger.info().log("Delta: \(gestureDelta)px")
 
         var events: [FBSimulatorHIDEvent] = []