From ad3581d1f3d042abfa311f9ce37328f02ac91067 Mon Sep 17 00:00:00 2001 From: onevcat Date: Tue, 28 Jul 2026 10:21:25 +0900 Subject: [PATCH] feat: expose orientation in the Viewer snapshot screen payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward `data.orientation` from `describe-ui --json` (the calibrated interface orientation, carried since #38) into the `screen` object of `GET /api/snapshot`, instead of leaving the SPA to spot the `(landscape-right)` suffix in the outline text. The key is omitted whenever the CLI reports none (Android, legacy daemons), matching the CLI envelope semantics. The SPA `Screen` type gains the optional `orientation` field, and the toolbar tags a rotated screen next to the W×H readout — portrait earns no tag, mirroring the outline header behaviour. Closes #57 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011M3T94iAdsJisr6oH7ynof Signed-off-by: onevcat --- CHANGELOG.md | 1 + Sources/SimUse/Viewer/ViewerAPIHandlers.swift | 11 ++++++++- Tests/ViewerAPIHandlersTests.swift | 23 +++++++++++++++++++ Tools/Viewer/README.md | 2 +- Tools/Viewer/src/App.tsx | 9 +++++++- Tools/Viewer/src/types.ts | 9 ++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8822d8b..9e90396 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SIM_USE_HID_TRANSPORT=indigo|dtuhid` debug override to force a specific iOS HID transport (default: automatic per-boot selection). The per-UDID daemon keeps the environment it was spawned with — combine with `SIM_USE_NO_DAEMON=1` or restart the daemon for ad-hoc experiments. - `describe-ui --no-raw` (top-level, `ios describe-ui`, and `android describe-ui`): with `--json`, omit the raw accessibility tree from the envelope. `data.raw` typically dominates the payload on real app screens and is only useful for debugging sim-use itself; `outline` / `entries` / `lists` are unaffected. - *Keeping output small* section in `skills/sim-use/SKILL.md`: steers agents to prefer the text outline, pair `--json` with `--no-raw`, verify via outline instead of screenshots, reuse the verify read as the next observe, and batch known sequences. +- Viewer: `GET /api/snapshot` now forwards the CLI's calibrated interface orientation as `screen.orientation` (omitted when `describe-ui` reports none — Android, legacy daemons), and the SPA tags a rotated screen next to the W×H readout, e.g. `874×402 (landscape-right)`. (#57) ### Changed diff --git a/Sources/SimUse/Viewer/ViewerAPIHandlers.swift b/Sources/SimUse/Viewer/ViewerAPIHandlers.swift index 1c18ebc..ebcef7f 100644 --- a/Sources/SimUse/Viewer/ViewerAPIHandlers.swift +++ b/Sources/SimUse/Viewer/ViewerAPIHandlers.swift @@ -83,7 +83,16 @@ struct ViewerAPIHandlers { } let data = (envelope["data"] as? [String: Any]) ?? [:] let outline = data["outline"] as? String - let screen = parseScreenFromOutline(outline) + var screen = parseScreenFromOutline(outline) + // `describe-ui --json` carries the calibrated interface + // orientation whenever calibration ran (issue #38); forward + // it verbatim rather than parsing the `(landscape-right)` + // suffix back out of the outline header. Absent on Android + // and legacy daemons — the key is simply omitted then, + // matching the CLI envelope semantics. + if let orientation = data["orientation"] as? String { + screen?["orientation"] = orientation + } var payload: [String: Any] = [ "ok": true, "capturedAt": iso8601Now(), diff --git a/Tests/ViewerAPIHandlersTests.swift b/Tests/ViewerAPIHandlersTests.swift index b4bc5ea..d29fd99 100644 --- a/Tests/ViewerAPIHandlersTests.swift +++ b/Tests/ViewerAPIHandlersTests.swift @@ -240,5 +240,28 @@ struct ViewerAPIHandlersTests { #expect(screen["appLabel"] as? String == "SampleApp") #expect(screen["width"] as? Int == 874) #expect(screen["height"] as? Int == 402) + // No `data.orientation` in the envelope → no `orientation` + // key in the screen payload (issue #57). + #expect(screen["orientation"] == nil) + } + + @Test("snapshot: forwards data.orientation into the screen payload") + func snapshotForwardsOrientation() async throws { + let envelope = """ + {"ok":true,"data":{"platform":"ios","orientation":"landscape-right","outline":"App: SampleApp 874x402 (landscape-right)\\n\\n[Top y<120]\\n","entries":[],"lists":[]}} + """ + let (handlers, cleanup) = try makeHandlers(stdout: envelope, exitCode: 0) + defer { cleanup() } + + let response = await handlers.snapshot(getRequest(query: ["deviceId": "TEST-UDID"])) + + #expect(response.status == 200) + let body = try jsonBody(response) + let screen = try #require(body["screen"] as? [String: Any]) + #expect(screen["orientation"] as? String == "landscape-right") + // The rest of the screen payload is unaffected. + #expect(screen["appLabel"] as? String == "SampleApp") + #expect(screen["width"] as? Int == 874) + #expect(screen["height"] as? Int == 402) } } diff --git a/Tools/Viewer/README.md b/Tools/Viewer/README.md index 64e501b..9050050 100644 --- a/Tools/Viewer/README.md +++ b/Tools/Viewer/README.md @@ -38,7 +38,7 @@ make viewer # or: scripts/build-viewer.sh ## API endpoints - `GET /api/devices` — list booted simulators and connected Android devices. -- `GET /api/snapshot?deviceId=` — `sim-use ui --json` → `{ screen, entries, outline, capturedAt }`. `udid=` is still accepted as a deprecated alias. +- `GET /api/snapshot?deviceId=` — `sim-use ui --json` → `{ screen, entries, outline, capturedAt }`. `screen.orientation` carries the CLI's calibrated interface orientation (`portrait`, `landscape-right`, …) and is omitted when the CLI reports none (Android). `udid=` is still accepted as a deprecated alias. - `POST /api/tap` — replay `sim-use tap @N` on the selected element. ## Controls diff --git a/Tools/Viewer/src/App.tsx b/Tools/Viewer/src/App.tsx index 2d1db8a..7900342 100644 --- a/Tools/Viewer/src/App.tsx +++ b/Tools/Viewer/src/App.tsx @@ -322,7 +322,14 @@ export default function App() { )} {snapshot?.screen && ( - {snapshot.screen.width}×{snapshot.screen.height} ·{" "} + {snapshot.screen.width}×{snapshot.screen.height} + {/* Mirror the outline header: portrait is the default + and earns no tag, only a rotated screen shows one. */} + {snapshot.screen.orientation && + snapshot.screen.orientation !== "portrait" && ( + <> ({snapshot.screen.orientation}) + )}{" "} + ·{" "} {matchIds ? `${matchIds.size} / ${snapshot.entries.length} match` : `${snapshot.entries.length} elements`} diff --git a/Tools/Viewer/src/types.ts b/Tools/Viewer/src/types.ts index 1bdd98a..f905ef3 100644 --- a/Tools/Viewer/src/types.ts +++ b/Tools/Viewer/src/types.ts @@ -56,6 +56,15 @@ export interface Screen { appLabel: string; width: number; height: number; + /** + * Calibrated interface orientation forwarded from `describe-ui + * --json` (`data.orientation`): `"portrait"`, + * `"portrait-upside-down"`, `"landscape-right"` or + * `"landscape-left"`. Absent when the CLI didn't calibrate + * (Android, legacy daemons). Widened to `string` for + * forward-compat, like `Device.platform`. + */ + orientation?: string; } export interface Snapshot {