From e76436f76f839964f0252509936ccdae6ec8942d Mon Sep 17 00:00:00 2001 From: onevcat Date: Tue, 28 Jul 2026 16:52:40 +0900 Subject: [PATCH 1/2] feat: opt-in ui coordinate space for explicit swipe/touch coordinates Explicit swipe/touch coordinates keep their device-native portrait default - the issue #34 acceptance contract - and gain --coordinate-space ui to opt endpoints into the visual space describe-ui prints. In ui mode the endpoints ride a per-command orientation calibration (the shared UISpaceCalibrationLoader, reused by the gesture presets from the previous commit); a failed calibration degrades to native dispatch with an advisory in the envelope. Native mode stays zero-cost: no tree fetch, no calibration, and no advisory noise. touch accepts ui in the atomic --down --up form only: a split touch could straddle a rotation and its two halves would calibrate independently into different spaces. Batch swipe/touch steps share the batch-wide calibration via the new BatchContext.uiSpaceCalibration helper (the gesture step converges on it too), and an unreachable AX tree records a step-prefixed advisory instead of failing the batch. The top-level cross-platform commands accept the flag everywhere and ignore it on Android, where display coordinates already rotate with the UI. TDD: CoordinateSpaceTests written first against the flag-only stub - 7 failures (split-touch rejection, batch ui calibration for swipe and touch, fetch-failure advisory) with the native zero-cost cases green, then implemented to a full pass. 1190 tests green. Closes #66 (PR B; PR A made directional gesture presets orientation-aware) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Rir4WwrPShf5DYzziJ8RyC Signed-off-by: onevcat --- CHANGELOG.md | 1 + README.md | 2 +- Sources/SimUse/Commands/Swipe.swift | 4 + Sources/SimUse/Commands/Touch.swift | 7 +- Sources/SimUseCore/CoordinateSpace.swift | 18 ++ .../A11y/GestureOrientationMapping.swift | 24 +++ .../iOSSimBackend/Batch/BatchContext.swift | 15 ++ .../Batch/Command+BatchConvertible.swift | 46 +++-- .../Verbs/IOSSimGestureCommand.swift | 14 +- .../Verbs/IOSSimSwipeCommand.swift | 45 ++++- .../Verbs/IOSSimTouchCommand.swift | 58 +++++- Tests/CommandAdvisoryContractTests.swift | 27 +++ Tests/CoordinateSpaceTests.swift | 183 ++++++++++++++++++ skills/sim-use/SKILL.md | 2 +- 14 files changed, 405 insertions(+), 41 deletions(-) create mode 100644 Sources/SimUseCore/CoordinateSpace.swift create mode 100644 Tests/CoordinateSpaceTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d1b7ed..b677b882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `swipe` / `touch --coordinate-space native|ui` (top-level and `ios` subcommands): explicit coordinates keep their device-native portrait default, and `ui` opts them into the visual space `describe-ui` prints — endpoints are orientation-calibrated per command (per batch for batch steps), so outline coordinates stay correct on a rotated device (#66). `touch` accepts `ui` in the atomic `--down --up` form only: a split touch could straddle a rotation and land its two halves in different spaces. A failed calibration degrades to native dispatch and says so via the `advisory` envelope key. On Android the flag is accepted and ignored (display coordinates already rotate with the UI). - The agent-eval suite can now pin exactly which sim-use binary a run evaluates: `make eval ARGS="-b "` / `scripts/eval.sh --sim-use ` / `run.py --sim-use ` (default remains whatever `sim-use` resolves to on PATH). The wrapper and runner print `sim-use under test: ()` up front and the report header records it, so a run can never silently exercise the wrong binary — and development builds under `.build/` can be evaluated directly. New repo skill `.claude/skills/run-evals/` orchestrates the whole flow for agents and contributors: environment prep (Device Hub closed, fixtures installed), binary selection, cost confirmation, and verdict triage; `e2e/agent-evals/README.md` documents the new prereqs and flags. - `SIM_USE_HID_TRANSPORT=indigo|dtuhid` debug override to force a specific iOS HID transport (default: automatic per-boot selection). The per-UDID daemon keeps the environment it was spawned with — combine with `SIM_USE_NO_DAEMON=1` or restart the daemon for ad-hoc experiments. - `SIM_USE_DEBUG=1` turns on the logger's stderr sink (with debug-level detail) on every construction path, including `ios batch` without `--verbose`, making internal info-lines — notably the HID transport-selection signals logged at connection creation — visible in normal runs and in the daemon logfile. Same daemon-environment caveat as `SIM_USE_HID_TRANSPORT`. (#67) diff --git a/README.md b/README.md index 6a125abc..47fe8b3c 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ sim-use gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --device $UDID `--pre-delay` / `--post-delay` / `--duration` work on `tap`, `swipe`, and `gesture` alike for coarse timing control. -Single-finger presets (`scroll-*`, `swipe-from-*-edge`) name **visual** directions and are orientation-aware on iOS: their canvas size and rotation are auto-detected per command, so `scroll-up` scrolls the on-screen content up whether the device is portrait, landscape, or upside-down. Explicit `swipe`/`touch` coordinates remain device-native portrait space by contract; pinch/rotate presets are likewise untransformed. +Single-finger presets (`scroll-*`, `swipe-from-*-edge`) name **visual** directions and are orientation-aware on iOS: their canvas size and rotation are auto-detected per command, so `scroll-up` scrolls the on-screen content up whether the device is portrait, landscape, or upside-down. Explicit `swipe`/`touch` coordinates remain device-native portrait space by default; pass `--coordinate-space ui` to give them in the visual space `ui` prints instead (orientation-calibrated per command; `touch` supports this in the atomic `--down --up` form only). Pinch/rotate presets remain untransformed. On Android the flag is accepted and ignored — display coordinates already rotate with the UI. ### Text input diff --git a/Sources/SimUse/Commands/Swipe.swift b/Sources/SimUse/Commands/Swipe.swift index fb55c3de..b52b09c8 100644 --- a/Sources/SimUse/Commands/Swipe.swift +++ b/Sources/SimUse/Commands/Swipe.swift @@ -21,6 +21,9 @@ struct Swipe: SimUseExecutableCommand { @OptionGroup var coordinates: SwipeCoordinateOptions + @Option(name: .customLong("coordinate-space"), help: "iOS only: 'native' (device-native portrait, the default) or 'ui' (visual space as printed by describe-ui; orientation-calibrated per command). Android coordinates are always display space, which already rotates with the UI — the flag is accepted and ignored there.") + var coordinateSpace: CoordinateSpace = .native + @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds.") var duration: Double? @@ -91,6 +94,7 @@ struct Swipe: SimUseExecutableCommand { func makeIOSSubcommand() -> IOSSimSwipeCommand { var sub = IOSSimSwipeCommand() sub.coordinates = coordinates + sub.coordinateSpace = coordinateSpace sub.duration = duration sub.delta = delta sub.preDelay = preDelay diff --git a/Sources/SimUse/Commands/Touch.swift b/Sources/SimUse/Commands/Touch.swift index 9fdc7822..ecdfe142 100644 --- a/Sources/SimUse/Commands/Touch.swift +++ b/Sources/SimUse/Commands/Touch.swift @@ -58,6 +58,9 @@ struct Touch: SimUseExecutableCommand { @Option(name: .customLong("delay"), help: "Delay between touch down and up events in seconds (if both are specified).") var delay: Double? + @Option(name: .customLong("coordinate-space"), help: "iOS only: 'native' (device-native portrait, the default) or 'ui' (visual space as printed by describe-ui; orientation-calibrated; atomic --down --up form only). Android coordinates are always display space, which already rotates with the UI — the flag is accepted and ignored there.") + var coordinateSpace: CoordinateSpace = .native + @OptionGroup var device: DeviceOptions @OptionGroup var json: JSONOutputOptions @@ -88,7 +91,8 @@ struct Touch: SimUseExecutableCommand { pointX: pointX, pointY: pointY, touchDown: touchDown, touchUp: touchUp, - delay: delay + delay: delay, + coordinateSpace: coordinateSpace ) } @@ -116,6 +120,7 @@ struct Touch: SimUseExecutableCommand { sub.pointY = pointY sub.touchDown = touchDown sub.touchUp = touchUp + sub.coordinateSpace = coordinateSpace sub.delay = delay sub.device = device sub.json = json diff --git a/Sources/SimUseCore/CoordinateSpace.swift b/Sources/SimUseCore/CoordinateSpace.swift new file mode 100644 index 00000000..2716c137 --- /dev/null +++ b/Sources/SimUseCore/CoordinateSpace.swift @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +import ArgumentParser + +/// Coordinate space for explicit gesture coordinates (issue #66). +/// +/// `native` — device-native portrait framebuffer points: the HID +/// layer's own space and the historical contract for explicit +/// coordinates (issue #34 acceptance criterion). Zero-cost: no AX +/// round-trip, no calibration. +/// +/// `ui` — visual space, the space every frame printed by `describe-ui` +/// uses. Endpoints are transformed through a per-command orientation +/// calibration before dispatch, so coordinates lifted from the outline +/// stay correct when the device is rotated. +public enum CoordinateSpace: String, CaseIterable, ExpressibleByArgument, Sendable { + case native + case ui +} diff --git a/Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift b/Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift index 7691e67b..fc64c569 100644 --- a/Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift +++ b/Sources/iOSSimBackend/A11y/GestureOrientationMapping.swift @@ -46,3 +46,27 @@ public enum GestureOrientationMapping { return (Double(start.x), Double(start.y), Double(end.x), Double(end.y)) } } + +/// Shared calibration loader for standalone verbs whose coordinates +/// run in ui space (directional gesture presets; `--coordinate-space +/// ui` on swipe/touch). Degrades to an identity dispatch with an +/// explicit advisory when the simulator cannot be calibrated at all — +/// never silently. +@MainActor +public enum UISpaceCalibrationLoader { + public static func load( + udid: String, + fallbackMessage: String, + logger: SimUseLogger + ) async -> OrientationCalibration { + do { + return try await AccessibilityFetcher.fetchOrientationCalibration(for: udid, logger: logger) + } catch { + logger.info().log("Orientation calibration unavailable (\(error.localizedDescription)); dispatching in native portrait axes") + return .identity(advisory: CommandAdvisory( + kind: .orientationCalibrationFallback, + message: fallbackMessage + )) + } + } +} diff --git a/Sources/iOSSimBackend/Batch/BatchContext.swift b/Sources/iOSSimBackend/Batch/BatchContext.swift index 291877a8..6dae39f6 100644 --- a/Sources/iOSSimBackend/Batch/BatchContext.swift +++ b/Sources/iOSSimBackend/Batch/BatchContext.swift @@ -124,4 +124,19 @@ public final class BatchContext { } return calibration } + + /// Batch-wide calibration for steps whose coordinates run in ui + /// space (directional gesture presets, `--coordinate-space ui`). + /// An unreachable AX tree degrades to an identity dispatch with a + /// recorded advisory instead of failing the whole batch. + public func uiSpaceCalibration(fallbackMessage: String, logger: SimUseLogger) async -> OrientationCalibration { + if let roots = try? await accessibilityRoots(logger: logger) { + return await orientationCalibration(roots: roots, logger: logger) + } + recordAdvisory(CommandAdvisory( + kind: .orientationCalibrationFallback, + message: fallbackMessage + )) + return .identity() + } } diff --git a/Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift b/Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift index 3472e105..e1b0c416 100644 --- a/Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift +++ b/Sources/iOSSimBackend/Batch/Command+BatchConvertible.swift @@ -106,9 +106,23 @@ extension IOSSimTapCommand: BatchConvertible { extension IOSSimSwipeCommand: BatchConvertible { public func toBatchPrimitives(context: BatchContext, logger: SimUseLogger) async throws -> [BatchPrimitive] { - let coords = try resolvedCoordinates() + let userCoords = try resolvedCoordinates() let swipeDuration = duration ?? 1.0 let swipeDelta = delta ?? 50.0 + + let coords: (startX: Double, startY: Double, endX: Double, endY: Double) + if coordinateSpace == .ui { + let calibration = await context.uiSpaceCalibration( + fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.", + logger: logger + ) + let start = calibration.hidPoint(x: userCoords.startX, y: userCoords.startY) + let end = calibration.hidPoint(x: userCoords.endX, y: userCoords.endY) + coords = (start.x, start.y, end.x, end.y) + } else { + coords = (userCoords.startX, userCoords.startY, userCoords.endX, userCoords.endY) + } + let swipeEvent = FBSimulatorHIDEvent.swipe( coords.startX, yStart: coords.startY, @@ -139,16 +153,10 @@ extension IOSSimGestureCommand: BatchConvertible { // with tap selector steps. An unreachable AX tree degrades // to an identity dispatch with a recorded advisory instead // of failing the whole batch. - let calibration: OrientationCalibration - if let roots = try? await context.accessibilityRoots(logger: logger) { - calibration = await context.orientationCalibration(roots: roots, logger: logger) - } else { - calibration = .identity() - context.recordAdvisory(CommandAdvisory( - kind: .orientationCalibrationFallback, - message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated." - )) - } + let calibration = await context.uiSpaceCalibration( + fallbackMessage: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated.", + logger: logger + ) let visual = GestureOrientationMapping.visualSize( explicitWidth: screenWidth, explicitHeight: screenHeight, calibration: calibration) let stroke = preset.strokes(screenWidth: visual.width, screenHeight: visual.height)[0] @@ -170,8 +178,20 @@ extension IOSSimGestureCommand: BatchConvertible { extension IOSSimTouchCommand: BatchConvertible { public func toBatchPrimitives(context: BatchContext, logger: SimUseLogger) async throws -> [BatchPrimitive] { - let touchDownEvent = FBSimulatorHIDEvent.touch(direction: .down, x: pointX, y: pointY) - let touchUpEvent = FBSimulatorHIDEvent.touch(direction: .up, x: pointX, y: pointY) + // ui space is atomic-form-only (rejected in validate for split + // steps), so one batch-wide calibration covers both halves. + let point: (x: Double, y: Double) + if coordinateSpace == .ui { + let calibration = await context.uiSpaceCalibration( + fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.", + logger: logger + ) + point = calibration.hidPoint(x: pointX, y: pointY) + } else { + point = (pointX, pointY) + } + let touchDownEvent = FBSimulatorHIDEvent.touch(direction: .down, x: point.x, y: point.y) + let touchUpEvent = FBSimulatorHIDEvent.touch(direction: .up, x: point.x, y: point.y) if touchDown && touchUp { let holdDelay = delay ?? 0.1 diff --git a/Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift b/Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift index f4c22eca..248b0dd4 100644 --- a/Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift +++ b/Sources/iOSSimBackend/Verbs/IOSSimGestureCommand.swift @@ -210,15 +210,11 @@ public struct IOSSimGestureCommand: SimUseExecutableCommand { preset: GesturePreset, logger: SimUseLogger ) async -> OrientationCalibration { - do { - return try await AccessibilityFetcher.fetchOrientationCalibration(for: udid, logger: logger) - } catch { - logger.info().log("Orientation calibration unavailable (\(error.localizedDescription)); dispatching \(preset.rawValue) in native portrait axes") - return .identity(advisory: CommandAdvisory( - kind: .orientationCalibrationFallback, - message: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated." - )) - } + await UISpaceCalibrationLoader.load( + udid: udid, + fallbackMessage: "Screen orientation could not be determined; '\(preset.rawValue)' was dispatched in device-native portrait axes and may point the wrong way if the device is rotated.", + logger: logger + ) } private func runSingleTouch( diff --git a/Sources/iOSSimBackend/Verbs/IOSSimSwipeCommand.swift b/Sources/iOSSimBackend/Verbs/IOSSimSwipeCommand.swift index d698e10d..05a6b6a1 100644 --- a/Sources/iOSSimBackend/Verbs/IOSSimSwipeCommand.swift +++ b/Sources/iOSSimBackend/Verbs/IOSSimSwipeCommand.swift @@ -13,11 +13,19 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand { /// Carries the resolved coordinates so `format(_:)` renders from /// the execution result instead of re-resolving the raw flags — /// same shape as `IOSSimTapCommand.ExecutionResult`. - public struct ExecutionResult: Codable { + public struct ExecutionResult: Codable, CommandAdvisoryProviding { public let coordinates: SwipeCoordinates + /// Excluded from the encoded `data` payload via `CodingKeys` + /// — the envelope hoists it to the top-level `advisory` key. + public var commandAdvisory: CommandAdvisory? = nil - public init(coordinates: SwipeCoordinates) { + public init(coordinates: SwipeCoordinates, commandAdvisory: CommandAdvisory? = nil) { self.coordinates = coordinates + self.commandAdvisory = commandAdvisory + } + + private enum CodingKeys: String, CodingKey { + case coordinates } } @@ -28,6 +36,9 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand { @OptionGroup public var coordinates: SwipeCoordinateOptions + @Option(name: .customLong("coordinate-space"), help: "Space the coordinates are given in: 'native' (device-native portrait, the default and historical contract) or 'ui' (visual space as printed by describe-ui; orientation-calibrated per command so outline coordinates stay correct on a rotated device).") + public var coordinateSpace: CoordinateSpace = .native + @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds.") public var duration: Double? @@ -127,6 +138,26 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand { logger.info().log("Performing swipe from (\(coords.startX), \(coords.startY)) to (\(coords.endX), \(coords.endY))") logger.info().log("Duration: \(swipeDuration)s, Delta: \(swipeDelta)px") + // Native (the default) dispatches the user's coordinates + // untouched — the historical contract. ui opts them into the + // visual space describe-ui prints (issue #66 PR B). + var advisory: CommandAdvisory? = nil + let dispatch: (startX: Double, startY: Double, endX: Double, endY: Double) + if coordinateSpace == .ui { + let calibration = await UISpaceCalibrationLoader.load( + udid: device.resolved, + fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.", + logger: logger + ) + advisory = calibration.advisory + let start = calibration.hidPoint(x: coords.startX, y: coords.startY) + let end = calibration.hidPoint(x: coords.endX, y: coords.endY) + dispatch = (start.x, start.y, end.x, end.y) + logger.info().log("Coordinates (HID space): (\(dispatch.startX), \(dispatch.startY)) to (\(dispatch.endX), \(dispatch.endY))") + } else { + dispatch = (coords.startX, coords.startY, coords.endX, coords.endY) + } + var events: [FBSimulatorHIDEvent] = [] if let preDelay, preDelay > 0 { @@ -135,10 +166,10 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand { } let swipeEvent = FBSimulatorHIDEvent.swipe( - coords.startX, - yStart: coords.startY, - xEnd: coords.endX, - yEnd: coords.endY, + dispatch.startX, + yStart: dispatch.startY, + xEnd: dispatch.endX, + yEnd: dispatch.endY, delta: swipeDelta, duration: swipeDuration ) @@ -158,6 +189,6 @@ public struct IOSSimSwipeCommand: SimUseExecutableCommand { ) logger.info().log("Swipe gesture completed successfully") - return ExecutionResult(coordinates: coords) + return ExecutionResult(coordinates: coords, commandAdvisory: advisory) } } diff --git a/Sources/iOSSimBackend/Verbs/IOSSimTouchCommand.swift b/Sources/iOSSimBackend/Verbs/IOSSimTouchCommand.swift index 42137d0f..aa43bc79 100644 --- a/Sources/iOSSimBackend/Verbs/IOSSimTouchCommand.swift +++ b/Sources/iOSSimBackend/Verbs/IOSSimTouchCommand.swift @@ -15,8 +15,16 @@ import SimUseCore /// open across other commands). The split form has no Android peer — /// the cross-platform forwarder rejects it on Android. public struct IOSSimTouchCommand: SimUseExecutableCommand { - public struct ExecutionResult: Codable { - public init() {} + public struct ExecutionResult: Codable, CommandAdvisoryProviding { + /// Excluded from the encoded `data` payload via `CodingKeys` + /// — the envelope hoists it to the top-level `advisory` key. + public var commandAdvisory: CommandAdvisory? = nil + + public init(commandAdvisory: CommandAdvisory? = nil) { + self.commandAdvisory = commandAdvisory + } + + private enum CodingKeys: CodingKey {} } public static let configuration = CommandConfiguration( @@ -51,6 +59,9 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { @Option(name: .customLong("delay"), help: "Delay between touch down and up events in seconds (if both are specified).") public var delay: Double? + @Option(name: .customLong("coordinate-space"), help: "Space the coordinates are given in: 'native' (device-native portrait, the default) or 'ui' (visual space as printed by describe-ui; orientation-calibrated). 'ui' requires the atomic --down --up form — the split form could straddle a rotation and land the two halves in different spaces.") + public var coordinateSpace: CoordinateSpace = .native + @OptionGroup public var device: DeviceOptions @OptionGroup public var json: JSONOutputOptions @@ -72,7 +83,8 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { pointX: pointX, pointY: pointY, touchDown: touchDown, touchUp: touchUp, - delay: delay + delay: delay, + coordinateSpace: coordinateSpace ) } @@ -84,7 +96,8 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { pointY: Double, touchDown: Bool, touchUp: Bool, - delay: Double? + delay: Double?, + coordinateSpace: CoordinateSpace = .native ) throws { guard pointX >= 0, pointY >= 0 else { throw ValidationError("Coordinates must be non-negative values.") @@ -94,6 +107,16 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { throw ValidationError("At least one of --down or --up must be specified.") } + // A split touch (--down and --up in separate invocations) can + // straddle a rotation: the two halves would calibrate + // independently and land in different spaces. Restrict ui + // coordinates to the atomic form. + if coordinateSpace == .ui { + guard touchDown && touchUp else { + throw ValidationError("--coordinate-space ui requires the atomic --down --up form. Use native coordinates for split touches.") + } + } + if let delay { guard delay >= 0 else { throw ValidationError("Delay must be non-negative.") @@ -114,6 +137,23 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { logger.info().log("Performing touch events at (\(pointX), \(pointY))") + // ui space is atomic-form-only (validated above), so a single + // calibration covers both halves of the touch. + var advisory: CommandAdvisory? = nil + let point: (x: Double, y: Double) + if coordinateSpace == .ui { + let calibration = await UISpaceCalibrationLoader.load( + udid: device.resolved, + fallbackMessage: "Screen orientation could not be determined; --coordinate-space ui coordinates were dispatched as device-native portrait and may be wrong if the device is rotated.", + logger: logger + ) + advisory = calibration.advisory + point = calibration.hidPoint(x: pointX, y: pointY) + logger.info().log("Coordinates (HID space): (\(point.x), \(point.y))") + } else { + point = (pointX, pointY) + } + if touchDown && touchUp { // Send down and up as separate HID submissions so iOS // recognizers observe a real hold duration for long-press @@ -122,7 +162,7 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { logger.info().log("Touch down") try await HIDInteractor.performHIDEvent( - FBSimulatorHIDEvent.touch(direction: .down, x: pointX, y: pointY), + FBSimulatorHIDEvent.touch(direction: .down, x: point.x, y: point.y), for: device.resolved, logger: logger ) @@ -134,27 +174,27 @@ public struct IOSSimTouchCommand: SimUseExecutableCommand { logger.info().log("Touch up") try await HIDInteractor.performHIDEvent( - FBSimulatorHIDEvent.touch(direction: .up, x: pointX, y: pointY), + FBSimulatorHIDEvent.touch(direction: .up, x: point.x, y: point.y), for: device.resolved, logger: logger ) } else if touchDown { logger.info().log("Touch down") try await HIDInteractor.performHIDEvent( - FBSimulatorHIDEvent.touch(direction: .down, x: pointX, y: pointY), + FBSimulatorHIDEvent.touch(direction: .down, x: point.x, y: point.y), for: device.resolved, logger: logger ) } else { logger.info().log("Touch up") try await HIDInteractor.performHIDEvent( - FBSimulatorHIDEvent.touch(direction: .up, x: pointX, y: pointY), + FBSimulatorHIDEvent.touch(direction: .up, x: point.x, y: point.y), for: device.resolved, logger: logger ) } logger.info().log("Touch events completed successfully") - return ExecutionResult() + return ExecutionResult(commandAdvisory: advisory) } } \ No newline at end of file diff --git a/Tests/CommandAdvisoryContractTests.swift b/Tests/CommandAdvisoryContractTests.swift index 4be5b7c0..18382ed6 100644 --- a/Tests/CommandAdvisoryContractTests.swift +++ b/Tests/CommandAdvisoryContractTests.swift @@ -35,6 +35,33 @@ struct CommandAdvisoryContractTests { #expect(decoded.y == 20) } + @Test("swipe result encodes without the advisory and decodes it as nil") + func swipeResult() throws { + let coords = SwipeCoordinates(startX: 1, startY: 2, endX: 3, endY: 4) + let result = IOSSimSwipeCommand.ExecutionResult(coordinates: coords, commandAdvisory: contractAdvisory) + #expect(result.commandAdvisory == contractAdvisory) + + let json = try encodedJSON(result) + #expect(!json.contains("advisory")) + #expect(json.contains(#""startX":1"#)) + + let decoded = try JSONDecoder().decode(IOSSimSwipeCommand.ExecutionResult.self, from: Data(json.utf8)) + #expect(decoded.commandAdvisory == nil) + #expect(decoded.coordinates.endY == 4) + } + + @Test("touch result encodes without the advisory and decodes it as nil") + func touchResult() throws { + let result = IOSSimTouchCommand.ExecutionResult(commandAdvisory: contractAdvisory) + #expect(result.commandAdvisory == contractAdvisory) + + let json = try encodedJSON(result) + #expect(json == #"{}"#) + + let decoded = try JSONDecoder().decode(IOSSimTouchCommand.ExecutionResult.self, from: Data(json.utf8)) + #expect(decoded.commandAdvisory == nil) + } + @Test("gesture result encodes without the advisory and decodes it as nil") func gestureResult() throws { let result = IOSSimGestureCommand.ExecutionResult(commandAdvisory: contractAdvisory) diff --git a/Tests/CoordinateSpaceTests.swift b/Tests/CoordinateSpaceTests.swift new file mode 100644 index 00000000..2835e4a9 --- /dev/null +++ b/Tests/CoordinateSpaceTests.swift @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 +@testable import iOSSimBackend +import Foundation +import SimUseCore +import Testing + +// PR B of issue #66: explicit swipe/touch coordinates keep their +// device-native portrait default (the issue #34 acceptance contract), +// and `--coordinate-space ui` opts endpoints into the visual space +// printed by describe-ui. These tests pin the opt-in rules: +// * native stays zero-cost — no tree fetch, no calibration; +// * ui rides the batch-wide calibration exactly like gesture presets; +// * ui + split touch is rejected — the two halves could straddle a +// rotation and land in different spaces. + +private let quietLogger = SimUseLogger(writeToStdErr: false) + +private func makeSmallTree() throws -> [AccessibilityElement] { + let json = """ + [{"type": "Application", "AXLabel": "App", "frame": {"x": 0, "y": 0, "width": 1376, "height": 1032}, "children": [ + {"type": "Button", "AXLabel": "Corner", "frame": {"x": 10, "y": 900, "width": 100, "height": 40}, "enabled": true} + ]}] + """ + return try JSONDecoder().decode([AccessibilityElement].self, from: Data(json.utf8)) +} + +@MainActor +private final class CalibratorSpy { + private(set) var calls = 0 + let result: OrientationCalibration + + init(result: OrientationCalibration) { + self.result = result + } + + func record() -> OrientationCalibration { + calls += 1 + return result + } +} + +@MainActor +private func makeContext(spy: CalibratorSpy, tree: [AccessibilityElement]) -> BatchContext { + BatchContext( + simulatorUDID: "FAKE-UDID", + axCachePolicy: .perBatch, + typeSubmissionMode: .chunked, + typeChunkSize: 200, + fetchElements: { _, _ in tree }, + calibrator: { _, _, _ in spy.record() } + ) +} + +@Suite("Touch — coordinate-space validation") +struct TouchCoordinateSpaceValidationTests { + + @Test("ui space with the atomic form validates") + func atomicUIValidates() throws { + try IOSSimTouchCommand.validateOptions( + pointX: 10, pointY: 10, touchDown: true, touchUp: true, + delay: nil, coordinateSpace: .ui) + } + + @Test("ui space with a split --down rejects") + func splitDownUIRejects() { + #expect(throws: (any Error).self) { + try IOSSimTouchCommand.validateOptions( + pointX: 10, pointY: 10, touchDown: true, touchUp: false, + delay: nil, coordinateSpace: .ui) + } + } + + @Test("ui space with a split --up rejects") + func splitUpUIRejects() { + #expect(throws: (any Error).self) { + try IOSSimTouchCommand.validateOptions( + pointX: 10, pointY: 10, touchDown: false, touchUp: true, + delay: nil, coordinateSpace: .ui) + } + } + + @Test("native space keeps the split form working") + func splitNativeStillValidates() throws { + try IOSSimTouchCommand.validateOptions( + pointX: 10, pointY: 10, touchDown: true, touchUp: false, + delay: nil, coordinateSpace: .native) + } +} + +@Suite("Batch — swipe/touch coordinate space") +@MainActor +struct BatchCoordinateSpaceTests { + private func parseStep(_ tokens: [String], context: BatchContext) async throws -> [BatchPrimitive] { + context.beginStep() + return try await BatchStepParser.parseStepTokens( + tokens, + globalUDID: "FAKE-UDID", + context: context, + logger: quietLogger + ) + } + + private var landscape: OrientationCalibration { + OrientationCalibration( + orientation: .landscapeRight, + native: NativePortraitSize(width: 1032, height: 1376), + probesUsed: 1, + advisory: nil + ) + } + + @Test("Native swipe steps stay zero-cost — no calibration") + func nativeSwipeSkipsCalibration() async throws { + let spy = CalibratorSpy(result: landscape) + let context = makeContext(spy: spy, tree: try makeSmallTree()) + + let primitives = try await parseStep( + ["swipe", "--from", "100,200", "--to", "300,400"], context: context) + + #expect(primitives.count == 1) + #expect(spy.calls == 0) + } + + @Test("ui swipe steps ride the batch-wide calibration") + func uiSwipeCalibrates() async throws { + let spy = CalibratorSpy(result: landscape) + let context = makeContext(spy: spy, tree: try makeSmallTree()) + + let primitives = try await parseStep( + ["swipe", "--from", "100,200", "--to", "300,400", "--coordinate-space", "ui"], + context: context) + + #expect(primitives.count == 1) + #expect(spy.calls == 1) + } + + @Test("ui touch steps ride the batch-wide calibration") + func uiTouchCalibrates() async throws { + let spy = CalibratorSpy(result: landscape) + let context = makeContext(spy: spy, tree: try makeSmallTree()) + + let primitives = try await parseStep( + ["touch", "-x", "100", "-y", "200", "--down", "--up", "--coordinate-space", "ui"], + context: context) + + #expect(primitives.count == 3) + #expect(spy.calls == 1) + } + + @Test("ui swipe with an unreachable tree degrades with a recorded advisory") + func uiSwipeFetchFailureDegrades() async throws { + let spy = CalibratorSpy(result: landscape) + let context = BatchContext( + simulatorUDID: "FAKE-UDID", + axCachePolicy: .perBatch, + typeSubmissionMode: .chunked, + typeChunkSize: 200, + fetchElements: { _, _ in throw CLIError(errorDescription: "no simulator") }, + calibrator: { _, _, _ in spy.record() } + ) + + let primitives = try await parseStep( + ["swipe", "--from", "100,200", "--to", "300,400", "--coordinate-space", "ui"], + context: context) + + #expect(primitives.count == 1) + #expect(spy.calls == 0) + #expect(context.commandAdvisories.count == 1) + #expect(context.commandAdvisories.first?.kind == .orientationCalibrationFallback) + } + + @Test("Split touch steps in ui space are rejected at parse time") + func splitTouchUIRejectsInBatch() async throws { + let spy = CalibratorSpy(result: landscape) + let context = makeContext(spy: spy, tree: try makeSmallTree()) + + await #expect(throws: (any Error).self) { + _ = try await parseStep( + ["touch", "-x", "100", "-y", "200", "--down", "--coordinate-space", "ui"], + context: context) + } + } +} diff --git a/skills/sim-use/SKILL.md b/skills/sim-use/SKILL.md index 57270f51..922d145c 100644 --- a/skills/sim-use/SKILL.md +++ b/skills/sim-use/SKILL.md @@ -98,7 +98,7 @@ Quick symptom index — see `references/pitfalls.md` for detailed recipes. | Android: `paste` denied | Background clipboard access blocked | Use `type` instead | | Outline shows `U+FFFC` in label | iOS icon placeholder character | Match with `--label-regex` excluding the prefix | | `[i] … covers ~N% of the screen` warning (text output, or `--json` top-level `advisory` key) | The selector resolved to a near-full-screen wrapper (common on Flutter/canvas UIs) and the tap hit its center, likely missing the intended control | Re-run `ui` and target the control via `@N`/`#`, or pass explicit `-x/-y`/`--point` | -| `[i] Screen orientation could not be confirmed…` / `…coordinates may be stale…` advisory | Device/app is rotated (the `App:` header shows a tag like `(landscape-right)`) and orientation self-calibration couldn't verify the mapping, or the `@N` snapshot predates a rotation | Re-run `ui` and tap again; selectors handle rotation automatically once calibration succeeds. Explicit `-x/-y`/`--point` is always device-native portrait space | +| `[i] Screen orientation could not be confirmed…` / `…coordinates may be stale…` advisory | Device/app is rotated (the `App:` header shows a tag like `(landscape-right)`) and orientation self-calibration couldn't verify the mapping, or the `@N` snapshot predates a rotation | Re-run `ui` and tap again; selectors handle rotation automatically once calibration succeeds. Explicit `-x/-y`/`--point` is device-native portrait space by default — on `swipe`/`touch`, pass `--coordinate-space ui` to use outline (visual-space) coordinates on a rotated device | ## 3. Crash awareness From 520e50199efab9589a9fa11d69a784578f8b82f4 Mon Sep 17 00:00:00 2001 From: onevcat Date: Tue, 28 Jul 2026 17:16:19 +0900 Subject: [PATCH 2/2] fix: accept --coordinate-space on the android direct commands Review follow-up: the top-level swipe/touch help promises the flag is accepted and ignored on Android, and every android direct command mirrors its top-level/iOS flag surface - but android swipe/touch rejected it at parse time. Declare the option on both as a documented no-op (Android coordinates are display space and already rotate with the UI) and pin the parity with parser tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Rir4WwrPShf5DYzziJ8RyC Signed-off-by: onevcat --- CHANGELOG.md | 2 +- .../Verbs/AndroidSwipeCommand.swift | 3 ++ .../Verbs/AndroidTouchCommand.swift | 3 ++ Tests/CoordinateSpaceTests.swift | 30 +++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c78a5186..e81d784a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `swipe` / `touch --coordinate-space native|ui` (top-level and `ios` subcommands): explicit coordinates keep their device-native portrait default, and `ui` opts them into the visual space `describe-ui` prints — endpoints are orientation-calibrated per command (per batch for batch steps), so outline coordinates stay correct on a rotated device (#66). `touch` accepts `ui` in the atomic `--down --up` form only: a split touch could straddle a rotation and land its two halves in different spaces. A failed calibration degrades to native dispatch and says so via the `advisory` envelope key. On Android the flag is accepted and ignored (display coordinates already rotate with the UI). +- `swipe` / `touch --coordinate-space native|ui` (top-level and `ios` subcommands): explicit coordinates keep their device-native portrait default, and `ui` opts them into the visual space `describe-ui` prints — endpoints are orientation-calibrated per command (per batch for batch steps), so outline coordinates stay correct on a rotated device (#66). `touch` accepts `ui` in the atomic `--down --up` form only: a split touch could straddle a rotation and land its two halves in different spaces. A failed calibration degrades to native dispatch and says so via the `advisory` envelope key. On Android the flag is accepted and ignored — on the top-level commands and the `android swipe` / `android touch` direct commands alike (display coordinates already rotate with the UI). - The agent-eval suite can now pin exactly which sim-use binary a run evaluates: `make eval ARGS="-b "` / `scripts/eval.sh --sim-use ` / `run.py --sim-use ` (default remains whatever `sim-use` resolves to on PATH). The wrapper and runner print `sim-use under test: ()` up front and the report header records it, so a run can never silently exercise the wrong binary — and development builds under `.build/` can be evaluated directly. New repo skill `.claude/skills/run-evals/` orchestrates the whole flow for agents and contributors: environment prep (Device Hub closed, fixtures installed), binary selection, cost confirmation, and verdict triage; `e2e/agent-evals/README.md` documents the new prereqs and flags. - `SIM_USE_HID_TRANSPORT=indigo|dtuhid` debug override to force a specific iOS HID transport (default: automatic per-boot selection). The per-UDID daemon keeps the environment it was spawned with — combine with `SIM_USE_NO_DAEMON=1` or restart the daemon for ad-hoc experiments. - `SIM_USE_DEBUG=1` turns on the logger's stderr sink (with debug-level detail) on every construction path, including `ios batch` without `--verbose`, making internal info-lines — notably the HID transport-selection signals logged at connection creation — visible in normal runs and in the daemon logfile. Same daemon-environment caveat as `SIM_USE_HID_TRANSPORT`. (#67) diff --git a/Sources/AndroidBackend/Verbs/AndroidSwipeCommand.swift b/Sources/AndroidBackend/Verbs/AndroidSwipeCommand.swift index 6e82e54d..7a7d236f 100644 --- a/Sources/AndroidBackend/Verbs/AndroidSwipeCommand.swift +++ b/Sources/AndroidBackend/Verbs/AndroidSwipeCommand.swift @@ -23,6 +23,9 @@ public struct AndroidSwipeCommand: SimUseExecutableCommand { @OptionGroup public var coordinates: SwipeCoordinateOptions + @Option(name: .customLong("coordinate-space"), help: "Accepted for parity with the iOS surface and ignored: Android coordinates are always display space, which already rotates with the UI.") + public var coordinateSpace: CoordinateSpace = .native + @Option(name: .customLong("duration"), help: "Duration of the swipe in seconds (default 0.3).") public var duration: Double = 0.3 diff --git a/Sources/AndroidBackend/Verbs/AndroidTouchCommand.swift b/Sources/AndroidBackend/Verbs/AndroidTouchCommand.swift index 4773deca..fab36a99 100644 --- a/Sources/AndroidBackend/Verbs/AndroidTouchCommand.swift +++ b/Sources/AndroidBackend/Verbs/AndroidTouchCommand.swift @@ -33,6 +33,9 @@ public struct AndroidTouchCommand: SimUseExecutableCommand { @Option(name: .customLong("delay"), help: "Hold duration in seconds between down and up (default 0.1).") public var delay: Double? + @Option(name: .customLong("coordinate-space"), help: "Accepted for parity with the iOS surface and ignored: Android coordinates are always display space, which already rotates with the UI.") + public var coordinateSpace: CoordinateSpace = .native + @Flag(name: .customLong("json"), help: "Emit the unified `{ok, data: {}}` envelope on success.") public var jsonOutput: Bool = false diff --git a/Tests/CoordinateSpaceTests.swift b/Tests/CoordinateSpaceTests.swift index 2835e4a9..1026cf1c 100644 --- a/Tests/CoordinateSpaceTests.swift +++ b/Tests/CoordinateSpaceTests.swift @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +@testable import AndroidBackend @testable import iOSSimBackend import Foundation import SimUseCore @@ -181,3 +182,32 @@ struct BatchCoordinateSpaceTests { } } } + +// The top-level commands promise that --coordinate-space is accepted +// everywhere and ignored on Android, and every Android direct command +// mirrors its top-level/iOS flag surface — so `sim-use android +// swipe/touch` must parse the flag too (as a no-op). + +@Suite("Android — coordinate-space parser parity") +struct AndroidCoordinateSpaceParityTests { + + @Test("android swipe parses --coordinate-space", arguments: ["native", "ui"]) + func androidSwipeParsesFlag(value: String) throws { + let parsed = try AndroidSwipeCommand.parseAsRoot( + ["--from", "1,1", "--to", "2,2", "--coordinate-space", value] + ) as? AndroidSwipeCommand + let command = try #require(parsed) + #expect(command.coordinateSpace.rawValue == value) + let coords = try command.coordinates.resolve() + #expect(coords.startX == 1 && coords.endY == 2) + } + + @Test("android touch parses --coordinate-space", arguments: ["native", "ui"]) + func androidTouchParsesFlag(value: String) throws { + let parsed = try AndroidTouchCommand.parseAsRoot( + ["-x", "1", "-y", "1", "--down", "--up", "--coordinate-space", value] + ) as? AndroidTouchCommand + let command = try #require(parsed) + #expect(command.coordinateSpace.rawValue == value) + } +}