feat: drive development-signed apps on physical iOS devices

CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `sim-use ios-device` (experimental): drive a development-signed app on a physical iPhone or iPad. `devices` lists attached devices, `ui` prints the foreground app's accessibility tree, and `tap --label` / `--label-contains` sends Activate to one unambiguous match. sim-use installs and signs no runner and needs no Developer Disk Image; the device must be unlocked and the target app must have `get-task-allow=true`. This channel intentionally omits coordinate tap, swipe and gesture because the daemon exposes no element geometry.
+
+### Fixed
+
+- Top-level and `sim-use ios` verbs now reject a physical iOS device UDID at resolution time with a pointer to `sim-use ios-device`, instead of misclassifying it as an Android serial and diagnosing a plugged-in iPhone as "not reachable via adb".
+- Wait for physical-device attachment notifications to settle so multiple USB-connected iOS devices are all discovered.
+- Reject empty physical-device hierarchies, missing or ambiguous tap targets, and invalid hierarchy concurrency instead of reporting success or silently choosing an element.
+- Correlate DTX replies by both identifier and conversation index so unsolicited device events cannot satisfy an unrelated pending request.
 - `record-video --gif-markers` (all three surfaces): bracket a GIF with START/END marker cards (~1 s each) so the forever-looping clip has a visible boundary. Opt-in — the default output remains a faithful capture of the screen. A failed card render degrades to a marker-less GIF instead of failing the transcode.
 
 ## [0.13.0] - 2026-08-06

Package.swift

@@ -48,7 +48,7 @@ let privateModuleMapFlags: [String] = ["-Xcc", "-I\(privateHeadersDir)"] + [
 }
 
 let fbLinkerFlags: [String] = [
-    "FBControlCore", "FBSimulatorControl", "XCTestBootstrap",
+    "FBControlCore", "FBSimulatorControl", "XCTestBootstrap", "FBDeviceControl",
 ].flatMap {
     [
         "-Xlinker", "-force_load",
@@ -87,6 +87,10 @@ let package = Package(
             name: "iOSSimBackend",
             targets: ["iOSSimBackend"]
         ),
+        .library(
+            name: "iOSDeviceBackend",
+            targets: ["iOSDeviceBackend"]
+        ),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
@@ -133,6 +137,21 @@ let package = Package(
             ],
             plugins: ["VersionPlugin"]
         ),
+        // Physical iOS devices, driven over the accessibility audit daemon
+        // (usbmux lockdown -> DTX). sim-use installs no XCUITest runner and
+        // needs no Developer Disk Image; the target app must already be
+        // development-signed. With no element geometry, this target cannot
+        // share iOSSimBackend's frame-based machinery.
+        .target(
+            name: "iOSDeviceBackend",
+            dependencies: [
+                "SimUseCore",
+                "FBDeviceControl",
+                "FBControlCore",
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+            ],
+            path: "Sources/iOSDeviceBackend"
+        ),
         .target(
             name: "AndroidBackend",
             dependencies: [
@@ -159,9 +178,11 @@ let package = Package(
                 "SimUseVideo",
                 "AndroidBackend",
                 "iOSSimBackend",
+                "iOSDeviceBackend",
                 "FBSimulatorControl",
                 "FBControlCore",
                 "XCTestBootstrap",
+                "FBDeviceControl",
                 "CompanionUtilities"
             ],
             path: "Sources/SimUse",
@@ -188,7 +209,7 @@ let package = Package(
         ),
         .testTarget(
             name: "SimUseTests",
-            dependencies: ["SimUse", "iOSSimBackend", "SimUseCore", "SimUseVideo"],
+            dependencies: ["SimUse", "iOSSimBackend", "iOSDeviceBackend", "SimUseCore", "SimUseVideo"],
             path: "Tests",
             // `Tests/` is the umbrella path; the sub-target test
             // directories below sit under it as separate testTargets.
@@ -246,5 +267,9 @@ let package = Package(
             name: "CompanionUtilities",
             path: "build_products/XCFrameworks/CompanionUtilities.xcframework"
         ),
+        .binaryTarget(
+            name: "FBDeviceControl",
+            path: "build_products/XCFrameworks/FBDeviceControl.xcframework"
+        ),
     ]
 )

README.md

@@ -41,6 +41,7 @@ Plan, code, **verify**, ship — teach this CLI to your agent and close the last
 - [Install](#install)
 - [Platforms](#platforms)
 - [Commands](#commands)
+- [Physical iOS devices](#physical-ios-devices)
 - [Architecture](#architecture)
 - [Viewer](#viewer)
 - [Contributing](#contributing)
@@ -145,6 +146,8 @@ sim-use drives both **iOS Simulators** and **Android devices / emulators** throu
 
 For Android, run `sim-use android init --device <serial>` once to install the bridge APK. See `AGENTS.md` for Android toolchain setup.
 
+**Development-signed apps on physical iPhones and iPads** are reachable under a separate, experimental `sim-use ios-device` surface. sim-use installs and signs no runner and needs no Developer Disk Image; the connected device must be unlocked and the foreground app must have `get-task-allow=true`. That channel exposes no element geometry, so it trades coordinate taps, swipes and gestures for accessibility actions. See [Physical iOS devices](#physical-ios-devices).
+
 
 ## Commands
 
@@ -153,6 +156,7 @@ All device-scoped commands accept `--device <ID>` (optional when only one simula
   * **Top-level** — cross-platform verbs: `ui`, `tap`, `long-press`, `swipe`, `touch`, `multi-touch`, `type`, `paste`, `button`, `gesture`, `keyboard-state`, `screenshot`, `record-video`, `stream-video`, `app-state`. Same flags on iOS and Android.
   * **`sim-use ios <verb>`** — iOS-only: `key`, `key-combo`, `key-sequence`, `batch`.
   * **`sim-use android <verb>`** — Android-only: `init`, `devices`, `ping`, `scroll`.
+  * **`sim-use ios-device <verb>`** — physical iOS devices (experimental): `devices`, `ui`, `tap`. Separate from the top-level verbs because the capabilities differ — see [Physical iOS devices](#physical-ios-devices).
 
 Run `sim-use --help` or `sim-use <command> --help` for the full flag set.
 
@@ -383,9 +387,55 @@ SIM_USE_NO_DAEMON=1 sim-use ui --device $UDID
 Daemons self-exit after 600 s of idle and log to `/tmp/sim-use-<uid>/<UDID>.log`. Streaming commands (`screenshot`, `record-video`, `stream-video`) always run in-process regardless.
 
 
+## Physical iOS devices
+
+> **Experimental:** this surface intentionally supports development-signed target apps only. Its commands and compatibility may change while the device matrix grows.
+
+A connected iPhone or iPad is driven through its accessibility audit daemon over usbmux lockdown. sim-use installs no XCUITest runner, performs no signing and needs no Developer Disk Image. The foreground target app must already be signed with a Development provisioning profile whose final code-sign entitlements contain `get-task-allow=true`, and the device must be paired, trusted, unlocked and in Developer Mode. A Release-configuration build remains supported when installed with a Development profile.
+
+Distribution/Ad Hoc, TestFlight, App Store and system apps do not expose the hierarchy or actions required by this channel. `ui` and `tap` fail with an entitlement-oriented diagnostic instead of reporting an empty tree or a successful action.
+
+Verify the app before installing it when in doubt:
+
+```bash
+codesign -d --entitlements :- /path/to/MyApp.app
+# ... <key>get-task-allow</key><true/> ...
+```
+
+```bash
+sim-use ios-device devices
+# 00008140-000210603A40801C  My iPhone  iOS 27.0  Booted
+
+sim-use ios-device ui --device 00008140-000210603A40801C
+# Button  "Chats Button, Selected"
+# Button  "Friends"
+# ...
+# 117 elements (316 nodes) in 6647 ms
+
+sim-use ios-device tap --label "Friends" --element-type Button \
+  --device 00008140-000210603A40801C
+# Sent Activate to Friends Button
+
+# Dynamic labels can use the regular substring selector vocabulary.
+sim-use ios-device tap --label-contains "Reply" --element-type Button
+```
+
+A device is addressed by UDID or ECID, and `--device` is optional only when exactly one is attached. Run `ui` again after every action: accessibility actions are fire-and-forget, so the follow-up read is the authoritative verification.
+
+Passing a physical device UDID to a top-level or `sim-use ios` verb fails fast with a pointer back to this surface — those verbs only drive iOS Simulators and Android devices.
+
+This channel deliberately differs from the simulator backend:
+
+  * **No element geometry.** There is no coordinate tap, `swipe`, `gesture` or `multi-touch`. Only the exposed `tap` accessibility action is currently supported; unsupported simulator verbs are not routed here.
+  * **No cross-process aliases.** Element handles encode a live pointer and expire with their DTX connection. The outline therefore does not advertise `@N`; `tap` re-resolves `--label` or `--label-contains` in the same session that sends Activate.
+  * **Text output only.** The experimental `ios-device` commands do not yet support `--json`, screenshot or recording.
+  * **Slower snapshots.** A full tree costs a few seconds. `ui --fast` stops at labelled elements and is roughly 40% quicker, at the cost of about a quarter of the elements.
+  * **Reading order, not screen order.** With no frames to sort by, the outline follows accessibility nesting and reading order.
+
+
 ## Architecture
 
-sim-use drives iOS Simulators through the lower-level XCFrameworks of Facebook's [idb](https://github.com/facebook/idb) (statically linked), Apple's Accessibility APIs, and the simulator HID pipeline. Android devices are driven through an on-device bridge APK that exposes the AccessibilityService tree and input injection over HTTP, tunnelled via `adb forward`. Everything ships as a single binary; every command supports `--json` for machine consumption.
+sim-use drives iOS Simulators through the lower-level XCFrameworks of Facebook's [idb](https://github.com/facebook/idb) (statically linked), Apple's Accessibility APIs, and the simulator HID pipeline. Android devices are driven through an on-device bridge APK that exposes the AccessibilityService tree and input injection over HTTP, tunnelled via `adb forward`. Physical iOS devices go through a third path: idb's `FBDeviceControl` opens a lockdown service connection, over which sim-use speaks Apple's DTX message protocol to the accessibility audit daemon. Everything ships as a single binary. The established simulator and Android surfaces support `--json`; the experimental `ios-device` commands currently emit text only.
 
 
 ## Viewer

Sources/SimUse/main.swift

@@ -7,6 +7,7 @@ import Darwin
 import SimUseCore
 import AndroidBackend
 import iOSSimBackend
+import iOSDeviceBackend
 
 // MARK: - Main Entry Point
 //
@@ -101,6 +102,7 @@ struct SimUse: AsyncParsableCommand {
             // `IOSSimCommand` only — the top-level surface only carries
             // verbs that work on both platforms.
             IOSSimCommand.self,
+            IOSDeviceCommand.self,
             AndroidCommand.self,
         ]
     )

Sources/SimUseCore/Options/DeviceOptions.swift

@@ -58,7 +58,14 @@ public struct DeviceOptions: ParsableArguments {
             resolved = arg
             return
         }
-        resolved = try DeviceResolver.resolve(explicit: explicit)
+        let candidate = try DeviceResolver.resolve(explicit: explicit)
+        // Checked on the resolved value, not just the explicit flag, so a
+        // physical UDID arriving via SIM_USE_DEVICE / SIM_USE_UDID is
+        // rejected identically.
+        guard !PlatformRouter.looksLikePhysicalIOSDevice(candidate) else {
+            throw PhysicalIOSDeviceError(identifier: candidate)
+        }
+        resolved = candidate
     }
 
     /// Apply the same `--device` / `--udid` mutual-exclusion rule used

Sources/SimUseCore/PlatformRouter.swift

@@ -41,11 +41,29 @@ public enum PlatformRouter {
         return udid.range(of: pattern, options: .regularExpression) != nil
     }
 
+    /// `true` when the UDID looks like a physical iOS device identifier:
+    /// modern 8-16 hex (`00008130-00066D2A10EB8D3A`, iPhone XS and later)
+    /// or legacy 40-hex (iPhone X and earlier). Physical devices are
+    /// served by the separate `sim-use ios-device` surface, so `resolve`
+    /// maps neither shape to a platform; the shape exists so device
+    /// resolution can reject early with a pointer there instead of
+    /// misreading the modern shape as an Android serial.
+    public static func looksLikePhysicalIOSDevice(_ udid: String) -> Bool {
+        let trimmed = udid.trimmingCharacters(in: .whitespacesAndNewlines)
+        let modern = "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}$"
+        let legacy = "^[0-9A-Fa-f]{40}$"
+        return trimmed.range(of: modern, options: .regularExpression) != nil
+            || trimmed.range(of: legacy, options: .regularExpression) != nil
+    }
+
     /// `true` when the UDID looks like an Android serial.
     ///
     /// Heuristic, in order:
     ///   1. `emulator-…` prefix → always Android.
-    ///   2. iOS Simulator UDID shape → never Android.
+    ///   2. iOS Simulator or physical iOS device UDID shape → never
+    ///      Android. Without the physical exclusion the modern
+    ///      8-16-hex device UDID clears rule 3 and a plugged-in iPhone
+    ///      is diagnosed as an unreachable adb serial.
     ///   3. ASCII-only, length 4–32, allowed `[A-Za-z0-9._:-]`, with at
     ///      least one digit → Android.
     ///
@@ -58,6 +76,7 @@ public enum PlatformRouter {
         if trimmed.isEmpty { return false }
         if trimmed.hasPrefix("emulator-") { return true }
         if looksLikeIOSSim(trimmed) { return false }
+        if looksLikePhysicalIOSDevice(trimmed) { return false }
         guard trimmed.count >= 4, trimmed.count <= 32 else { return false }
         let allowed: (Character) -> Bool = { ch in
             ch.isASCII && (

Sources/SimUseCore/Types/Errors.swift

@@ -22,4 +22,26 @@ public struct CLIError: LocalizedError {
     public init(errorDescription: String) {
         self.errorDescription = errorDescription
     }
+}
+
+/// The target identifier names a physical iPhone or iPad, which the
+/// simulator and Android backends cannot serve. Thrown during device
+/// resolution so every UDID-scoped verb rejects before dispatch with a
+/// pointer to the experimental `ios-device` surface, instead of the
+/// shape heuristics misreading the modern 8-16-hex device UDID as an
+/// unreachable Android serial.
+public struct PhysicalIOSDeviceError: LocalizedError, HintProviding {
+    public let identifier: String
+
+    public init(identifier: String) {
+        self.identifier = identifier
+    }
+
+    public var errorDescription: String? {
+        "\(identifier) looks like a physical iOS device; this command only drives iOS Simulators and Android devices/emulators."
+    }
+
+    public var hint: String? {
+        "Physical iPhones and iPads use the experimental 'sim-use ios-device' surface: 'ios-device devices' lists attached devices, 'ios-device ui' reads the foreground app's accessibility tree, 'ios-device tap' activates an element by label. See 'sim-use ios-device --help'."
+    }
 }

Sources/iOSDeviceBackend/AXAudit/AXAuditAttribute.swift

@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: Apache-2.0
+import Foundation
+
+/// A request descriptor for `deviceElement:valueForAttribute:` and
+/// `deviceElement:performAction:withValue:`.
+///
+/// The daemon advertises the attributes it supports per inspector section, but
+/// the advertised list is not exhaustive — `Traits` is absent from it yet reads
+/// back fine — so the names below are the ones measured to work rather than the
+/// ones announced. `Frame` is deliberately missing: iOS serves no geometry on
+/// this channel at all (Xcode's own Accessibility Inspector shows no Frame row
+/// for a physical device either).
+public struct AXAuditAttribute: Equatable, Sendable {
+    public let name: String
+    public let humanReadableName: String
+    public let performsAction: Bool
+    public let isSettable: Bool
+    public let valueType: Int
+
+    public init(
+        name: String,
+        humanReadableName: String? = nil,
+        performsAction: Bool = false,
+        isSettable: Bool = false,
+        valueType: Int = 0
+    ) {
+        self.name = name
+        self.humanReadableName = humanReadableName ?? name
+        self.performsAction = performsAction
+        self.isSettable = isSettable
+        self.valueType = valueType
+    }
+
+    public var encoded: AXAuditValue {
+        .object(tag: "AXAuditElementAttribute_v1", fields: [
+            "AttributeNameValue_v1": .scalar(name),
+            "HumanReadableNameValue_v1": .scalar(humanReadableName),
+            "DisplayAsTree_v1": .scalar(0),
+            "IsInternal_v1": .scalar(0),
+            "PerformsActionValue_v1": .scalar(performsAction ? 1 : 0),
+            "SettableValue_v1": .scalar(isSettable ? 1 : 0),
+            "ValueTypeValue_v1": .scalar(valueType),
+        ])
+    }
+}
+
+// MARK: - Readable attributes
+
+public extension AXAuditAttribute {
+    static let label = AXAuditAttribute(name: "Label")
+    static let value = AXAuditAttribute(name: "Value")
+    static let hint = AXAuditAttribute(name: "Hint")
+    static let identifier = AXAuditAttribute(name: "Identifier")
+    static let traits = AXAuditAttribute(name: "Traits")
+    static let traitsHumanReadable = AXAuditAttribute(name: "TraitsHumanReadable")
+    static let className = AXAuditAttribute(name: "ElementClassName")
+    static let hierarchy = AXAuditAttribute(name: "_AXHierarchyElementsAttribute")
+}
+
+// MARK: - Actions
+
+public extension AXAuditAttribute {
+    /// Tapping. The numeric codes are the daemon's own action identifiers,
+    /// read back from the Actions section of a focus event.
+    static let activate = action(code: 2010, humanReadableName: "Activate")
+    static let scrollDown = action(code: 2006, humanReadableName: "Scroll down")
+    static let scrollUp = action(code: 2007, humanReadableName: "Scroll up")
+
+    static func action(code: Int, humanReadableName: String) -> AXAuditAttribute {
+        AXAuditAttribute(
+            name: "AXAction-\(code)",
+            humanReadableName: humanReadableName,
+            performsAction: true,
+            valueType: 1
+        )
+    }
+
+    /// App-defined actions (LINE's `Pin chat`, `Delete`, …). Their names embed a
+    /// live pointer, so they are only valid for the session that reported them.
+    static func custom(name: String, humanReadableName: String) -> AXAuditAttribute {
+        AXAuditAttribute(name: name, humanReadableName: humanReadableName, performsAction: true, valueType: 1)
+    }
+
+    var isScrollAction: Bool { self == .scrollDown || self == .scrollUp }
+}