feat: drive physical iOS devices over the accessibility audit daemon

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `sim-use ios-device` (experimental): drive a **physical** iPhone or iPad. `devices` lists attached devices, `ui` prints an outline of the foreground app's accessibility tree, and `tap --text` activates a matching element. The device is reached through its accessibility audit daemon over usbmux lockdown, so nothing is installed on it, nothing is code signed, and no Developer Disk Image is needed — but the device must be unlocked. This channel exposes no element geometry, so there is no coordinate tap, swipe or gesture: interaction goes through accessibility actions, and the outline is ordered by the accessibility reading order rather than by screen position.
+
 ## [0.13.0] - 2026-08-06
 
 ### Added

Package.swift

@@ -48,7 +48,7 @@ let privateModuleMapFlags: [String] = ["-Xcc", "-I\(privateHeadersDir)"] + [
 }
 
 let fbLinkerFlags: [String] = [
-    "FBControlCore", "FBSimulatorControl", "XCTestBootstrap",
+    "FBControlCore", "FBSimulatorControl", "XCTestBootstrap", "FBDeviceControl",
 ].flatMap {
     [
         "-Xlinker", "-force_load",
@@ -87,6 +87,10 @@ let package = Package(
             name: "iOSSimBackend",
             targets: ["iOSSimBackend"]
         ),
+        .library(
+            name: "iOSDeviceBackend",
+            targets: ["iOSDeviceBackend"]
+        ),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
@@ -133,6 +137,20 @@ let package = Package(
             ],
             plugins: ["VersionPlugin"]
         ),
+        // Physical iOS devices, driven over the accessibility audit daemon
+        // (usbmux lockdown -> DTX). No XCUITest runner, so no code signing and
+        // no Developer Disk Image — but also no element geometry, which is why
+        // this target does not share iOSSimBackend's frame-based machinery.
+        .target(
+            name: "iOSDeviceBackend",
+            dependencies: [
+                "SimUseCore",
+                "FBDeviceControl",
+                "FBControlCore",
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+            ],
+            path: "Sources/iOSDeviceBackend"
+        ),
         .target(
             name: "AndroidBackend",
             dependencies: [
@@ -159,9 +177,11 @@ let package = Package(
                 "SimUseVideo",
                 "AndroidBackend",
                 "iOSSimBackend",
+                "iOSDeviceBackend",
                 "FBSimulatorControl",
                 "FBControlCore",
                 "XCTestBootstrap",
+                "FBDeviceControl",
                 "CompanionUtilities"
             ],
             path: "Sources/SimUse",
@@ -188,7 +208,7 @@ let package = Package(
         ),
         .testTarget(
             name: "SimUseTests",
-            dependencies: ["SimUse", "iOSSimBackend", "SimUseCore", "SimUseVideo"],
+            dependencies: ["SimUse", "iOSSimBackend", "iOSDeviceBackend", "SimUseCore", "SimUseVideo"],
             path: "Tests",
             // `Tests/` is the umbrella path; the sub-target test
             // directories below sit under it as separate testTargets.
@@ -246,5 +266,9 @@ let package = Package(
             name: "CompanionUtilities",
             path: "build_products/XCFrameworks/CompanionUtilities.xcframework"
         ),
+        .binaryTarget(
+            name: "FBDeviceControl",
+            path: "build_products/XCFrameworks/FBDeviceControl.xcframework"
+        ),
     ]
 )

README.md

@@ -41,6 +41,7 @@ Plan, code, **verify**, ship — teach this CLI to your agent and close the last
 - [Install](#install)
 - [Platforms](#platforms)
 - [Commands](#commands)
+- [Physical iOS devices](#physical-ios-devices)
 - [Architecture](#architecture)
 - [Viewer](#viewer)
 - [Contributing](#contributing)
@@ -145,6 +146,8 @@ sim-use drives both **iOS Simulators** and **Android devices / emulators** throu
 
 For Android, run `sim-use android init --device <serial>` once to install the bridge APK. See `AGENTS.md` for Android toolchain setup.
 
+**Physical iPhones and iPads** are reachable too, under a separate `sim-use ios-device` surface — nothing is installed on the device, nothing is code signed and no Developer Disk Image is needed, but the device must be unlocked. That channel exposes no element geometry, so it trades coordinate taps, swipes and gestures for accessibility actions. See [Physical iOS devices](#physical-ios-devices).
+
 
 ## Commands
 
@@ -153,6 +156,7 @@ All device-scoped commands accept `--device <ID>` (optional when only one simula
   * **Top-level** — cross-platform verbs: `ui`, `tap`, `long-press`, `swipe`, `touch`, `multi-touch`, `type`, `paste`, `button`, `gesture`, `keyboard-state`, `screenshot`, `record-video`, `stream-video`, `app-state`. Same flags on iOS and Android.
   * **`sim-use ios <verb>`** — iOS-only: `key`, `key-combo`, `key-sequence`, `batch`.
   * **`sim-use android <verb>`** — Android-only: `init`, `devices`, `ping`, `scroll`.
+  * **`sim-use ios-device <verb>`** — physical iOS devices (experimental): `devices`, `ui`, `tap`. Separate from the top-level verbs because the capabilities differ — see [Physical iOS devices](#physical-ios-devices).
 
 Run `sim-use --help` or `sim-use <command> --help` for the full flag set.
 
@@ -381,9 +385,37 @@ SIM_USE_NO_DAEMON=1 sim-use ui --device $UDID
 Daemons self-exit after 600 s of idle and log to `/tmp/sim-use-<uid>/<UDID>.log`. Streaming commands (`screenshot`, `record-video`, `stream-video`) always run in-process regardless.
 
 
+## Physical iOS devices
+
+**Experimental.** A connected iPhone or iPad is driven through its accessibility audit daemon over usbmux lockdown — no XCUITest runner is installed, nothing is code signed, and no Developer Disk Image is mounted. The device must be **unlocked**; a locked screen accepts the connection and then reports no elements.
+
+```bash
+sim-use ios-device devices
+# 00008140-000210603A40801C  My iPhone  iOS 27.0  Booted
+
+sim-use ios-device ui
+# @4   Button  "Chats Button, Selected"
+# @5   Button  "Friends"
+# ...
+# 117 elements (316 nodes) in 6647 ms
+
+sim-use ios-device tap --text "Friends"
+# ✓ Activated Friends Button
+```
+
+A device is addressed by UDID or ECID, and `--device` is optional when only one is attached.
+
+This channel behaves differently from the simulator one, in ways that shape how you drive it:
+
+  * **No element geometry.** iOS serves no frames here — Xcode's own Accessibility Inspector shows no Frame row for a physical device either. So there is no coordinate tap, `swipe`, `gesture` or `multi-touch`; interaction is by accessibility action instead. Scrolling and app-defined swipe actions (a chat row's *Pin* or *Mute*, say) are reachable that way without any gesture.
+  * **No `@N` aliases across processes.** Element handles encode a live pointer and die with the connection, so `tap` resolves its target by `--text` inside a single session rather than reusing an alias from a previous `ui`.
+  * **Slower.** The daemon serialises requests, so a full tree costs a few seconds. `--fast` stops descending at labelled elements — roughly 40% quicker for about a quarter fewer elements.
+  * **Reading order, not screen order.** With no frames to sort by, the outline follows the accessibility tree's nesting and reading order.
+
+
 ## Architecture
 
-sim-use drives iOS Simulators through the lower-level XCFrameworks of Facebook's [idb](https://github.com/facebook/idb) (statically linked), Apple's Accessibility APIs, and the simulator HID pipeline. Android devices are driven through an on-device bridge APK that exposes the AccessibilityService tree and input injection over HTTP, tunnelled via `adb forward`. Everything ships as a single binary; every command supports `--json` for machine consumption.
+sim-use drives iOS Simulators through the lower-level XCFrameworks of Facebook's [idb](https://github.com/facebook/idb) (statically linked), Apple's Accessibility APIs, and the simulator HID pipeline. Android devices are driven through an on-device bridge APK that exposes the AccessibilityService tree and input injection over HTTP, tunnelled via `adb forward`. Physical iOS devices go through a third path: idb's `FBDeviceControl` opens a lockdown service connection, over which sim-use speaks Apple's DTX message protocol to the accessibility audit daemon. Everything ships as a single binary; every command supports `--json` for machine consumption.
 
 
 ## Viewer

Sources/SimUse/main.swift

@@ -7,6 +7,7 @@ import Darwin
 import SimUseCore
 import AndroidBackend
 import iOSSimBackend
+import iOSDeviceBackend
 
 // MARK: - Main Entry Point
 //
@@ -101,6 +102,7 @@ struct SimUse: AsyncParsableCommand {
             // `IOSSimCommand` only — the top-level surface only carries
             // verbs that work on both platforms.
             IOSSimCommand.self,
+            IOSDeviceCommand.self,
             AndroidCommand.self,
         ]
     )

Sources/iOSDeviceBackend/AXAudit/AXAuditAttribute.swift

@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: Apache-2.0
+import Foundation
+
+/// A request descriptor for `deviceElement:valueForAttribute:` and
+/// `deviceElement:performAction:withValue:`.
+///
+/// The daemon advertises the attributes it supports per inspector section, but
+/// the advertised list is not exhaustive — `Traits` is absent from it yet reads
+/// back fine — so the names below are the ones measured to work rather than the
+/// ones announced. `Frame` is deliberately missing: iOS serves no geometry on
+/// this channel at all (Xcode's own Accessibility Inspector shows no Frame row
+/// for a physical device either).
+public struct AXAuditAttribute: Equatable, Sendable {
+    public let name: String
+    public let humanReadableName: String
+    public let performsAction: Bool
+    public let isSettable: Bool
+    public let valueType: Int
+
+    public init(
+        name: String,
+        humanReadableName: String? = nil,
+        performsAction: Bool = false,
+        isSettable: Bool = false,
+        valueType: Int = 0
+    ) {
+        self.name = name
+        self.humanReadableName = humanReadableName ?? name
+        self.performsAction = performsAction
+        self.isSettable = isSettable
+        self.valueType = valueType
+    }
+
+    public var encoded: AXAuditValue {
+        .object(tag: "AXAuditElementAttribute_v1", fields: [
+            "AttributeNameValue_v1": .scalar(name),
+            "HumanReadableNameValue_v1": .scalar(humanReadableName),
+            "DisplayAsTree_v1": .scalar(0),
+            "IsInternal_v1": .scalar(0),
+            "PerformsActionValue_v1": .scalar(performsAction ? 1 : 0),
+            "SettableValue_v1": .scalar(isSettable ? 1 : 0),
+            "ValueTypeValue_v1": .scalar(valueType),
+        ])
+    }
+}
+
+// MARK: - Readable attributes
+
+public extension AXAuditAttribute {
+    static let label = AXAuditAttribute(name: "Label")
+    static let value = AXAuditAttribute(name: "Value")
+    static let hint = AXAuditAttribute(name: "Hint")
+    static let identifier = AXAuditAttribute(name: "Identifier")
+    static let traits = AXAuditAttribute(name: "Traits")
+    static let traitsHumanReadable = AXAuditAttribute(name: "TraitsHumanReadable")
+    static let className = AXAuditAttribute(name: "ElementClassName")
+    static let hierarchy = AXAuditAttribute(name: "_AXHierarchyElementsAttribute")
+}
+
+// MARK: - Actions
+
+public extension AXAuditAttribute {
+    /// Tapping. The numeric codes are the daemon's own action identifiers,
+    /// read back from the Actions section of a focus event.
+    static let activate = action(code: 2010, humanReadableName: "Activate")
+    static let scrollDown = action(code: 2006, humanReadableName: "Scroll down")
+    static let scrollUp = action(code: 2007, humanReadableName: "Scroll up")
+
+    static func action(code: Int, humanReadableName: String) -> AXAuditAttribute {
+        AXAuditAttribute(
+            name: "AXAction-\(code)",
+            humanReadableName: humanReadableName,
+            performsAction: true,
+            valueType: 1
+        )
+    }
+
+    /// App-defined actions (LINE's `Pin chat`, `Delete`, …). Their names embed a
+    /// live pointer, so they are only valid for the session that reported them.
+    static func custom(name: String, humanReadableName: String) -> AXAuditAttribute {
+        AXAuditAttribute(name: name, humanReadableName: humanReadableName, performsAction: true, valueType: 1)
+    }
+
+    var isScrollAction: Bool { self == .scrollDown || self == .scrollUp }
+}

Sources/iOSDeviceBackend/AXAudit/AXAuditClient.swift

@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: Apache-2.0
+import Foundation
+
+/// Sends one DTX selector and resolves its reply. Kept abstract so the
+/// selector layer can be exercised without a device attached.
+public protocol DTXInvoking: Sendable {
+    func invoke(_ selector: String, arguments: [AXAuditValue], expectsReply: Bool) async throws -> AXAuditValue
+}
+
+public extension DTXInvoking {
+    func invoke(_ selector: String, _ arguments: AXAuditValue...) async throws -> AXAuditValue {
+        try await invoke(selector, arguments: arguments, expectsReply: true)
+    }
+
+    func send(_ selector: String, _ arguments: AXAuditValue...) async throws {
+        _ = try await invoke(selector, arguments: arguments, expectsReply: false)
+    }
+}
+
+public enum AXAuditError: Error, LocalizedError, CustomStringConvertible {
+    case noRootElement
+    case unsupportedSelector(String)
+    case malformedReply(selector: String)
+
+    public var errorDescription: String? { description }
+
+    public var description: String {
+        switch self {
+        case .noRootElement:
+            return "the foreground app exposed no root accessibility element — unlock the device and try again"
+        case let .unsupportedSelector(selector):
+            return "the device does not support '\(selector)'"
+        case let .malformedReply(selector):
+            return "the device did not answer '\(selector)' as expected — this is what a locked screen looks like; unlock it and try again"
+        }
+    }
+}
+
+/// Selector-level API for `com.apple.accessibility.axAuditDaemon.remoteserver`.
+///
+/// Reachable over plain usbmux lockdown: no runner app, no code signing and no
+/// Developer Disk Image. Element geometry is not available on this channel at
+/// any iOS version measured, so there is intentionally no frame accessor.
+public struct AXAuditClient: Sendable {
+    private let transport: any DTXInvoking
+
+    public init(transport: any DTXInvoking) {
+        self.transport = transport
+    }
+
+    /// Must run before anything else: without it the daemon answers element
+    /// queries with nil even though the connection is healthy.
+    public func prepare() async throws {
+        try await transport.send("deviceInspectorEnable:", .bool(true))
+        try await transport.send("deviceSetAppMonitoringEnabled:", .bool(true))
+        try await transport.send("deviceInspectorShowVisuals:", .bool(false))
+    }
+
+    /// Releases the daemon's inspector state. Skipping this leaves the device
+    /// wedged: later sessions connect and answer, but every element query comes
+    /// back empty until something calls it. Abruptly dropping several sessions
+    /// at once is enough to trigger that.
+    public func finish() async {
+        try? await transport.send("deviceInspectorEnable:", .bool(false))
+        try? await transport.send("deviceSetAppMonitoringEnabled:", .bool(false))
+        try? await transport.send("devicePerformFinalCleanup")
+    }
+
+    public func capabilities() async throws -> [String] {
+        let reply = try await transport.invoke("deviceCapabilities")
+        guard case let .list(values) = reply.unwrapped else {
+            throw AXAuditError.malformedReply(selector: "deviceCapabilities")
+        }
+        return values.compactMap(\.stringValue)
+    }
+
+    public func apiVersion() async throws -> Int? {
+        try await transport.invoke("deviceApiVersion").intValue
+    }
+
+    public func rootElement() async throws -> AXAuditElement {
+        let reply = try await transport.invoke("deviceFetchSpecialElement:", .int(0))
+        guard let element = AXAuditElement(payload: reply) else { throw AXAuditError.noRootElement }
+        return element
+    }
+
+    public func value(of attribute: AXAuditAttribute, for element: AXAuditElement) async throws -> AXAuditValue {
+        try await transport.invoke("deviceElement:valueForAttribute:", element.encoded, attribute.encoded)
+    }
+
+    public func string(_ attribute: AXAuditAttribute, for element: AXAuditElement) async throws -> String? {
+        try await value(of: attribute, for: element).stringValue
+    }
+
+    /// Direct children of `element`, already carrying their label and role so a
+    /// tree walk needs one round trip per node rather than one per attribute.
+    public func children(of element: AXAuditElement) async throws -> [AXAuditNode] {
+        let reply = try await value(of: .hierarchy, for: element)
+        return reply.descendants(taggedWith: "AXAuditNode_v1").compactMap { node in
+            guard let child = node["AuditElementValue_v1"].flatMap(AXAuditElement.init(payload:)) else { return nil }
+            return AXAuditNode(
+                element: child,
+                summary: node["HumanReadableDescriptionValue_v1"]?.stringValue ?? "",
+                role: node["HumanReadableRoleDescriptionValue_v1"]?.stringValue ?? "",
+                isIgnored: node["IsIgnoredValue_v1"]?.boolValue ?? false
+            )
+        }
+    }
+
+    public func perform(_ action: AXAuditAttribute, on element: AXAuditElement) async throws {
+        try await transport.send("deviceElement:performAction:withValue:", element.encoded, action.encoded, .int(0))
+    }
+
+    public func captureScreenshot() async throws -> DeviceScreenshot {
+        let reply = try await transport.invoke("deviceCaptureScreenshot")
+        guard let screenshot = DeviceScreenshot(payload: reply) else {
+            throw AXAuditError.malformedReply(selector: "deviceCaptureScreenshot")
+        }
+        return screenshot
+    }
+}
+
+/// A screenshot plus the display metadata the daemon returns alongside it —
+/// the only geometry this channel offers.
+public struct DeviceScreenshot: Sendable {
+    public let pngData: Data
+    public let nativeScale: Int?
+    public let rotationRadians: Int?
+
+    init?(payload: AXAuditValue) {
+        guard let data = payload["imageData"]?.dataValue, data.starts(with: Self.pngMagic) else { return nil }
+        pngData = data
+        nativeScale = payload["displayNativeScale"]?.intValue
+        rotationRadians = payload["rotationRadians"]?.intValue
+    }
+
+    private static let pngMagic: [UInt8] = [0x89, 0x50, 0x4E, 0x47]
+}