feat: opt-in ui coordinate space for explicit swipe/touch coordinates

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `swipe` / `touch --coordinate-space native|ui` (top-level and `ios` subcommands): explicit coordinates keep their device-native portrait default, and `ui` opts them into the visual space `describe-ui` prints — endpoints are orientation-calibrated per command (per batch for batch steps), so outline coordinates stay correct on a rotated device (#66). `touch` accepts `ui` in the atomic `--down --up` form only: a split touch could straddle a rotation and land its two halves in different spaces. A failed calibration degrades to native dispatch and says so via the `advisory` envelope key. On Android the flag is accepted and ignored — on the top-level commands and the `android swipe` / `android touch` direct commands alike (display coordinates already rotate with the UI).
 - The agent-eval suite can now pin exactly which sim-use binary a run evaluates: `make eval ARGS="-b <path>"` / `scripts/eval.sh --sim-use <path>` / `run.py --sim-use <path>` (default remains whatever `sim-use` resolves to on PATH). The wrapper and runner print `sim-use under test: <real path> (<version>)` up front and the report header records it, so a run can never silently exercise the wrong binary — and development builds under `.build/` can be evaluated directly. New repo skill `.claude/skills/run-evals/` orchestrates the whole flow for agents and contributors: environment prep (Device Hub closed, fixtures installed), binary selection, cost confirmation, and verdict triage; `e2e/agent-evals/README.md` documents the new prereqs and flags.
 - `SIM_USE_HID_TRANSPORT=indigo|dtuhid` debug override to force a specific iOS HID transport (default: automatic per-boot selection). The per-UDID daemon keeps the environment it was spawned with — combine with `SIM_USE_NO_DAEMON=1` or restart the daemon for ad-hoc experiments.
 - `SIM_USE_DEBUG=1` turns on the logger's stderr sink (with debug-level detail) on every construction path, including `ios batch` without `--verbose`, making internal info-lines — notably the HID transport-selection signals logged at connection creation — visible in normal runs and in the daemon logfile. Same daemon-environment caveat as `SIM_USE_HID_TRANSPORT`. (#67)

README.md

@@ -208,7 +208,7 @@ sim-use gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --device $UDID
 
 `--pre-delay` / `--post-delay` / `--duration` work on `tap`, `swipe`, and `gesture` alike for coarse timing control.
 
-Single-finger presets (`scroll-*`, `swipe-from-*-edge`) name **visual** directions and are orientation-aware on iOS: their canvas size and rotation are auto-detected per command, so `scroll-up` scrolls the on-screen content up whether the device is portrait, landscape, or upside-down. Explicit `swipe`/`touch` coordinates remain device-native portrait space by contract; pinch/rotate presets are likewise untransformed.
+Single-finger presets (`scroll-*`, `swipe-from-*-edge`) name **visual** directions and are orientation-aware on iOS: their canvas size and rotation are auto-detected per command, so `scroll-up` scrolls the on-screen content up whether the device is portrait, landscape, or upside-down. Explicit `swipe`/`touch` coordinates remain device-native portrait space by default; pass `--coordinate-space ui` to give them in the visual space `ui` prints instead (orientation-calibrated per command; `touch` supports this in the atomic `--down --up` form only). Pinch/rotate presets remain untransformed. On Android the flag is accepted and ignored — display coordinates already rotate with the UI.
 
 ### Text input
 

Sources/AndroidBackend/Verbs/AndroidSwipeCommand.swift

@@ -23,6 +23,9 @@ public struct AndroidSwipeCommand: SimUseExecutableCommand {
 
     @OptionGroup public var coordinates: SwipeCoordinateOptions
 
+    @Option(name: .customLong("coordinate-space"), help: "Accepted for parity with the iOS surface and ignored: Android coordinates are always display space, which already rotates with the UI.")
+    public var coordinateSpace: CoordinateSpace = .native
+
     @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds (default 0.3).")
     public var duration: Double = 0.3
 

Sources/AndroidBackend/Verbs/AndroidTouchCommand.swift

@@ -33,6 +33,9 @@ public struct AndroidTouchCommand: SimUseExecutableCommand {
     @Option(name: .customLong("delay"), help: "Hold duration in seconds between down and up (default 0.1).")
     public var delay: Double?
 
+    @Option(name: .customLong("coordinate-space"), help: "Accepted for parity with the iOS surface and ignored: Android coordinates are always display space, which already rotates with the UI.")
+    public var coordinateSpace: CoordinateSpace = .native
+
     @Flag(name: .customLong("json"), help: "Emit the unified `{ok, data: {}}` envelope on success.")
     public var jsonOutput: Bool = false
 

Sources/SimUse/Commands/Swipe.swift

@@ -21,6 +21,9 @@ struct Swipe: SimUseExecutableCommand {
 
     @OptionGroup var coordinates: SwipeCoordinateOptions
 
+    @Option(name: .customLong("coordinate-space"), help: "iOS only: 'native' (device-native portrait, the default) or 'ui' (visual space as printed by describe-ui; orientation-calibrated per command). Android coordinates are always display space, which already rotates with the UI — the flag is accepted and ignored there.")
+    var coordinateSpace: CoordinateSpace = .native
+
     @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds.")
     var duration: Double?
 
@@ -91,6 +94,7 @@ struct Swipe: SimUseExecutableCommand {
     func makeIOSSubcommand() -> IOSSimSwipeCommand {
         var sub = IOSSimSwipeCommand()
         sub.coordinates = coordinates
+        sub.coordinateSpace = coordinateSpace
         sub.duration = duration
         sub.delta = delta
         sub.preDelay = preDelay

Sources/SimUse/Commands/Touch.swift

@@ -58,6 +58,9 @@ struct Touch: SimUseExecutableCommand {
     @Option(name: .customLong("delay"), help: "Delay between touch down and up events in seconds (if both are specified).")
     var delay: Double?
 
+    @Option(name: .customLong("coordinate-space"), help: "iOS only: 'native' (device-native portrait, the default) or 'ui' (visual space as printed by describe-ui; orientation-calibrated; atomic --down --up form only). Android coordinates are always display space, which already rotates with the UI — the flag is accepted and ignored there.")
+    var coordinateSpace: CoordinateSpace = .native
+
     @OptionGroup var device: DeviceOptions
 
     @OptionGroup var json: JSONOutputOptions
@@ -88,7 +91,8 @@ struct Touch: SimUseExecutableCommand {
             pointX: pointX, pointY: pointY,
             touchDown: touchDown,
             touchUp: touchUp,
-            delay: delay
+            delay: delay,
+            coordinateSpace: coordinateSpace
         )
     }
 
@@ -116,6 +120,7 @@ struct Touch: SimUseExecutableCommand {
         sub.pointY = pointY
         sub.touchDown = touchDown
         sub.touchUp = touchUp
+        sub.coordinateSpace = coordinateSpace
         sub.delay = delay
         sub.device = device
         sub.json = json

Sources/SimUseCore/CoordinateSpace.swift

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+import ArgumentParser
+
+/// Coordinate space for explicit gesture coordinates (issue #66).
+///
+/// `native` — device-native portrait framebuffer points: the HID
+/// layer's own space and the historical contract for explicit
+/// coordinates (issue #34 acceptance criterion). Zero-cost: no AX
+/// round-trip, no calibration.
+///
+/// `ui` — visual space, the space every frame printed by `describe-ui`
+/// uses. Endpoints are transformed through a per-command orientation
+/// calibration before dispatch, so coordinates lifted from the outline
+/// stay correct when the device is rotated.
+public enum CoordinateSpace: String, CaseIterable, ExpressibleByArgument, Sendable {
+    case native
+    case ui
+}

Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift

@@ -46,3 +46,27 @@ public enum GestureOrientationMapping {
         return (Double(start.x), Double(start.y), Double(end.x), Double(end.y))
     }
 }
+
+/// Shared calibration loader for standalone verbs whose coordinates
+/// run in ui space (directional gesture presets; `--coordinate-space
+/// ui` on swipe/touch). Degrades to an identity dispatch with an
+/// explicit advisory when the simulator cannot be calibrated at all —
+/// never silently.
+@MainActor
+public enum UISpaceCalibrationLoader {
+    public static func load(
+        udid: String,
+        fallbackMessage: String,
+        logger: SimUseLogger
+    ) async -> OrientationCalibration {
+        do {
+            return try await AccessibilityFetcher.fetchOrientationCalibration(for: udid, logger: logger)
+        } catch {
+            logger.info().log("Orientation calibration unavailable (\(error.localizedDescription)); dispatching in native portrait axes")
+            return .identity(advisory: CommandAdvisory(
+                kind: .orientationCalibrationFallback,
+                message: fallbackMessage
+            ))
+        }
+    }
+}

Sources/iOSSimBackend/Batch/BatchContext.swift

@@ -124,4 +124,19 @@ public final class BatchContext {
         }
         return calibration
     }
+
+    /// Batch-wide calibration for steps whose coordinates run in ui
+    /// space (directional gesture presets, `--coordinate-space ui`).
+    /// An unreachable AX tree degrades to an identity dispatch with a
+    /// recorded advisory instead of failing the whole batch.
+    public func uiSpaceCalibration(fallbackMessage: String, logger: SimUseLogger) async -> OrientationCalibration {
+        if let roots = try? await accessibilityRoots(logger: logger) {
+            return await orientationCalibration(roots: roots, logger: logger)
+        }
+        recordAdvisory(CommandAdvisory(
+            kind: .orientationCalibrationFallback,
+            message: fallbackMessage
+        ))
+        return .identity()
+    }
 }

Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift

@@ -106,9 +106,23 @@ extension IOSSimTapCommand: BatchConvertible {
 
 extension IOSSimSwipeCommand: BatchConvertible {
     public func toBatchPrimitives(context: BatchContext, logger: SimUseLogger) async throws -> [BatchPrimitive] {
-        let coords = try resolvedCoordinates()
+        let userCoords = try resolvedCoordinates()
         let swipeDuration = duration ?? 1.0
         let swipeDelta = delta ?? 50.0
+
+        let coords: (startX: Double, startY: Double, endX: Double, endY: Double)
+        if coordinateSpace == .ui {
+            let calibration = await context.uiSpaceCalibration(
+                fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.",
+                logger: logger
+            )
+            let start = calibration.hidPoint(x: userCoords.startX, y: userCoords.startY)
+            let end = calibration.hidPoint(x: userCoords.endX, y: userCoords.endY)
+            coords = (start.x, start.y, end.x, end.y)
+        } else {
+            coords = (userCoords.startX, userCoords.startY, userCoords.endX, userCoords.endY)
+        }
+
         let swipeEvent = FBSimulatorHIDEvent.swipe(
             coords.startX,
             yStart: coords.startY,
@@ -139,16 +153,10 @@ extension IOSSimGestureCommand: BatchConvertible {
             // with tap selector steps. An unreachable AX tree degrades
             // to an identity dispatch with a recorded advisory instead
             // of failing the whole batch.
-            let calibration: OrientationCalibration
-            if let roots = try? await context.accessibilityRoots(logger: logger) {
-                calibration = await context.orientationCalibration(roots: roots, logger: logger)
-            } else {
-                calibration = .identity()
-                context.recordAdvisory(CommandAdvisory(
-                    kind: .orientationCalibrationFallback,
-                    message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated."
-                ))
-            }
+            let calibration = await context.uiSpaceCalibration(
+                fallbackMessage: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated.",
+                logger: logger
+            )
             let visual = GestureOrientationMapping.visualSize(
                 explicitWidth: screenWidth, explicitHeight: screenHeight, calibration: calibration)
             let stroke = preset.strokes(screenWidth: visual.width, screenHeight: visual.height)[0]
@@ -170,8 +178,20 @@ extension IOSSimGestureCommand: BatchConvertible {
 
 extension IOSSimTouchCommand: BatchConvertible {
     public func toBatchPrimitives(context: BatchContext, logger: SimUseLogger) async throws -> [BatchPrimitive] {
-        let touchDownEvent = FBSimulatorHIDEvent.touch(direction: .down, x: pointX, y: pointY)
-        let touchUpEvent = FBSimulatorHIDEvent.touch(direction: .up, x: pointX, y: pointY)
+        // ui space is atomic-form-only (rejected in validate for split
+        // steps), so one batch-wide calibration covers both halves.
+        let point: (x: Double, y: Double)
+        if coordinateSpace == .ui {
+            let calibration = await context.uiSpaceCalibration(
+                fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.",
+                logger: logger
+            )
+            point = calibration.hidPoint(x: pointX, y: pointY)
+        } else {
+            point = (pointX, pointY)
+        }
+        let touchDownEvent = FBSimulatorHIDEvent.touch(direction: .down, x: point.x, y: point.y)
+        let touchUpEvent = FBSimulatorHIDEvent.touch(direction: .up, x: point.x, y: point.y)
 
         if touchDown && touchUp {
             let holdDelay = delay ?? 0.1

Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift

@@ -210,15 +210,11 @@ public struct IOSSimGestureCommand: SimUseExecutableCommand {
         preset: GesturePreset,
         logger: SimUseLogger
     ) async -> OrientationCalibration {
-        do {
-            return try await AccessibilityFetcher.fetchOrientationCalibration(for: udid, logger: logger)
-        } catch {
-            logger.info().log("Orientation calibration unavailable (\(error.localizedDescription)); dispatching \(preset.rawValue) in native portrait axes")
-            return .identity(advisory: CommandAdvisory(
-                kind: .orientationCalibrationFallback,
-                message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated."
-            ))
-        }
+        await UISpaceCalibrationLoader.load(
+            udid: udid,
+            fallbackMessage: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated.",
+            logger: logger
+        )
     }
 
     private func runSingleTouch(

Sources/iOSSimBackend/Verbs/IOSSimSwipeCommand.swift

@@ -13,11 +13,19 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand {
     /// Carries the resolved coordinates so `format(_:)` renders from
     /// the execution result instead of re-resolving the raw flags —
     /// same shape as `IOSSimTapCommand.ExecutionResult`.
-    public struct ExecutionResult: Codable {
+    public struct ExecutionResult: Codable, CommandAdvisoryProviding {
         public let coordinates: SwipeCoordinates
+        /// Excluded from the encoded `data` payload via `CodingKeys`
+        /// — the envelope hoists it to the top-level `advisory` key.
+        public var commandAdvisory: CommandAdvisory? = nil
 
-        public init(coordinates: SwipeCoordinates) {
+        public init(coordinates: SwipeCoordinates, commandAdvisory: CommandAdvisory? = nil) {
             self.coordinates = coordinates
+            self.commandAdvisory = commandAdvisory
+        }
+
+        private enum CodingKeys: String, CodingKey {
+            case coordinates
         }
     }
 
@@ -28,6 +36,9 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand {
 
     @OptionGroup public var coordinates: SwipeCoordinateOptions
 
+    @Option(name: .customLong("coordinate-space"), help: "Space the coordinates are given in: 'native' (device-native portrait, the default and historical contract) or 'ui' (visual space as printed by describe-ui; orientation-calibrated per command so outline coordinates stay correct on a rotated device).")
+    public var coordinateSpace: CoordinateSpace = .native
+
     @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds.")
     public var duration: Double?
 
@@ -127,6 +138,26 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand {
         logger.info().log("Performing swipe from (\(coords.startX), \(coords.startY)) to (\(coords.endX), \(coords.endY))")
         logger.info().log("Duration: \(swipeDuration)s, Delta: \(swipeDelta)px")
 
+        // Native (the default) dispatches the user's coordinates
+        // untouched — the historical contract. ui opts them into the
+        // visual space describe-ui prints (issue #66 PR B).
+        var advisory: CommandAdvisory? = nil
+        let dispatch: (startX: Double, startY: Double, endX: Double, endY: Double)
+        if coordinateSpace == .ui {
+            let calibration = await UISpaceCalibrationLoader.load(
+                udid: device.resolved,
+                fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.",
+                logger: logger
+            )
+            advisory = calibration.advisory
+            let start = calibration.hidPoint(x: coords.startX, y: coords.startY)
+            let end = calibration.hidPoint(x: coords.endX, y: coords.endY)
+            dispatch = (start.x, start.y, end.x, end.y)
+            logger.info().log("Coordinates (HID space): (\(dispatch.startX), \(dispatch.startY)) to (\(dispatch.endX), \(dispatch.endY))")
+        } else {
+            dispatch = (coords.startX, coords.startY, coords.endX, coords.endY)
+        }
+
         var events: [FBSimulatorHIDEvent] = []
 
         if let preDelay, preDelay > 0 {
@@ -135,10 +166,10 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand {
         }
 
         let swipeEvent = FBSimulatorHIDEvent.swipe(
-            coords.startX,
-            yStart: coords.startY,
-            xEnd: coords.endX,
-            yEnd: coords.endY,
+            dispatch.startX,
+            yStart: dispatch.startY,
+            xEnd: dispatch.endX,
+            yEnd: dispatch.endY,
             delta: swipeDelta,
             duration: swipeDuration
         )
@@ -158,6 +189,6 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand {
         )
 
         logger.info().log("Swipe gesture completed successfully")
-        return ExecutionResult(coordinates: coords)
+        return ExecutionResult(coordinates: coords, commandAdvisory: advisory)
     }
 }