fix: encode stream-video frames once, into the container they claim

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- `stream-video` frames now match the container they are labelled with, and are encoded exactly once. Both platforms' screenshot-backed formats treated the default `--scale 1.0` / `--quality 80` as an unconditional passthrough while every MJPEG frame header claimed `image/jpeg`, so at default settings strict MJPEG clients were handed PNG payloads under a JPEG MIME type. Each format now declares the container its consumers can actually decode — `mjpeg` carries JPEG (ffmpeg's `mpjpeg` demuxer and IP-camera clients reject anything else), while `raw` and `ffmpeg` carry the capture's lossless PNG untouched, since `raw` delimits frames with its own length prefix and the documented `ffmpeg` pipeline sniffs the container via `-f image2pipe`. On iOS the capture is read off the framebuffer as a `CGImage` and encoded once into the format's container, replacing a PNG encode plus a decode plus a re-encode; on Android `screencap`'s PNG now reaches `raw`/`ffmpeg` with no transcode at all, and only `mjpeg` pays an encode. The iOS `record-video` screenshot fallback stopped round-tripping every frame through PNG as well. The run banner now reports the frame container (`Frames: jpeg q80`) in place of a `--quality` that never applied to the PNG formats, and `--quality`'s help says which formats it affects. (#94 — thanks @SunsetWan for the diagnosis and the MJPEG frame-parsing test helper!)
+
+
 ### Fixed
 
 - The bundled skill preflight now rejects an older `sim-use` CLI before device discovery and prints the Homebrew upgrade command, instead of misdiagnosing newly documented device types as disconnected.

README.md

@@ -330,6 +330,22 @@ sim-use record-video --device $UDID --format gif                      # sim-use-
 sim-use record-video --device $UDID --format gif --fps 15 --scale 0.4 --output demo.gif
 ```
 
+`stream-video`'s screenshot-backed formats each carry the container their
+consumers can actually decode, and the capture is encoded into it exactly
+once — no frame is transcoded just to change its label:
+
+| `--format` | frames | consumer |
+|---|---|---|
+| `mjpeg` | JPEG (`--quality`, default 80) | browsers, `ffmpeg -f mpjpeg`, MJPEG clients — none of which accept another container |
+| `ffmpeg` | PNG, lossless | `ffmpeg -f image2pipe`, which sniffs the container |
+| `raw` | PNG, lossless | your own reader; each frame carries a 4-byte big-endian length prefix |
+| `bgra` (iOS) / `h264` (Android) | raw pixels / H.264 Annex B | not screenshot-backed |
+
+`--quality` therefore only bites on `mjpeg`; the PNG formats are lossless,
+and trade roughly 7x the bytes per frame for it (measured on a booted
+iPhone 17 Pro home screen: 455 KB/frame at `mjpeg`'s default quality vs
+3.5 MB/frame as PNG). Reach for `mjpeg` when the stream leaves the machine.
+
 `record-video` captures a real H.264 stream and muxes it straight into the
 MP4 (passthrough — no per-frame screenshot re-encoding). iOS records at a
 constant `--fps` (default 30, max 60); Android records at the device's

Sources/AndroidBackend/Verbs/AndroidStreamVideoCommand.swift

@@ -28,6 +28,17 @@ public struct AndroidStreamVideoCommand: SimUseExecutableCommand {
         case raw
         case ffmpeg
         case h264
+
+        /// The container this format's consumers can decode, or nil for
+        /// `h264`, which is a byte passthrough. Matches the iOS surface —
+        /// `stream-video` promises byte-identical framing across platforms.
+        func frameContainer(quality: Int) -> FrameContainer? {
+            switch self {
+            case .mjpeg: .jpeg(quality: quality)
+            case .raw, .ffmpeg: .png
+            case .h264: nil
+            }
+        }
     }
 
     /// Summary of a completed stream run. The video bytes are written to
@@ -62,7 +73,7 @@ public struct AndroidStreamVideoCommand: SimUseExecutableCommand {
     @Option(help: "Frames per second for the JPEG formats (1-30, default: 10). Ignored by --format h264 (native variable frame rate).")
     public var fps: Int?
 
-    @Option(help: "JPEG quality for the JPEG formats / bitrate factor for h264 (1-100, default: 80)")
+    @Option(help: "JPEG quality for mjpeg / bitrate factor for h264 (1-100, default: 80). Ignored by raw/ffmpeg, which carry lossless PNG frames.")
     public var quality: Int = 80
 
     @Option(help: "Scale factor (0.1-1.0, default: 1.0)")
@@ -157,12 +168,15 @@ public struct AndroidStreamVideoCommand: SimUseExecutableCommand {
                 cancellationFlag: cancellationFlag
             )
         case .mjpeg, .raw, .ffmpeg:
-            return try await streamJPEGFrames(
+            guard let container = format.frameContainer(quality: quality) else {
+                throw CLIError(errorDescription: "Format \(format.rawValue) does not carry encoded frames")
+            }
+            return try await streamFrames(
                 adb: adb,
                 serial: serial,
                 format: format,
+                container: container,
                 fps: fps ?? 10,
-                quality: quality,
                 scale: scale,
                 cancellationFlag: cancellationFlag
             )
@@ -283,17 +297,17 @@ public struct AndroidStreamVideoCommand: SimUseExecutableCommand {
     /// `mjpeg` = multipart/x-mixed-replace with `--mjpegstream` boundaries,
     /// `raw` = 4-byte big-endian length prefix per frame,
     /// `ffmpeg` = bare concatenated frames.
-    private static func streamJPEGFrames(
+    private static func streamFrames(
         adb: Adb,
         serial: String,
         format: OutputFormat,
+        container: FrameContainer,
         fps: Int,
-        quality: Int,
         scale: Double,
         cancellationFlag: CancellationFlag
     ) async throws -> ExecutionResult {
         FileHandle.standardError.write(Data("Starting screencap-based video stream from Android device \(serial)...\n".utf8))
-        FileHandle.standardError.write(Data("Format: \(format.rawValue), FPS: \(fps), Quality: \(quality), Scale: \(scale)\n".utf8))
+        FileHandle.standardError.write(Data("Format: \(format.rawValue), FPS: \(fps), Frames: \(container), Scale: \(scale)\n".utf8))
         FileHandle.standardError.write(Data("Press Ctrl+C to stop streaming\n".utf8))
 
         let frameInterval = 1.0 / Double(fps)
@@ -318,11 +332,11 @@ public struct AndroidStreamVideoCommand: SimUseExecutableCommand {
 
             do {
                 let frameData = try AndroidRecordVideoCommand.captureAndroidScreencap(adbPath: adbPath, serial: serial)
-                let processedData = try await VideoFrameUtilities.processJPEGData(frameData, scale: scale, quality: quality)
+                let processedData = try VideoFrameUtilities.transcodeFrame(frameData, to: container, scale: scale)
 
                 switch format {
                 case .mjpeg:
-                    let frameHeader = "\(mjpegBoundary)\r\nContent-Type: image/jpeg\r\nContent-Length: \(processedData.count)\r\n\r\n"
+                    let frameHeader = "\(mjpegBoundary)\r\nContent-Type: \(container.mimeType)\r\nContent-Length: \(processedData.count)\r\n\r\n"
                     sink.write(Data(frameHeader.utf8))
                     sink.write(processedData)
                     sink.write(Data("\r\n".utf8))

Sources/SimUseVideo/VideoCommandSupport.swift

@@ -61,6 +61,61 @@ public struct VideoWriterStallError: Error, LocalizedError, Equatable {
     }
 }
 
+/// The wire container a captured frame is carried in.
+///
+/// Stream formats differ in what their consumers can actually decode, so
+/// each one names its container and the capture path encodes into it
+/// exactly once. Nothing re-containers bytes just to change their label.
+public enum FrameContainer: Equatable, Sendable, CustomStringConvertible {
+    /// Lossless. `--format raw` delimits frames itself and ffmpeg's
+    /// `image2pipe` demuxer sniffs the container, so both carry PNG
+    /// straight from the capture with no re-encode.
+    case png
+    /// Motion JPEG only accepts JPEG: ffmpeg's `mpjpeg` demuxer and the
+    /// IP-camera clients that read `multipart/x-mixed-replace` reject
+    /// every other container.
+    case jpeg(quality: Int)
+
+    public var mimeType: String {
+        switch self {
+        case .png: "image/png"
+        case .jpeg: "image/jpeg"
+        }
+    }
+
+    public var description: String {
+        switch self {
+        case .png: "png (lossless)"
+        case .jpeg(let quality): "jpeg q\(quality)"
+        }
+    }
+
+    /// Leading bytes identifying an encoded frame as this container.
+    var magicBytes: [UInt8] {
+        switch self {
+        case .png: [0x89, 0x50, 0x4E, 0x47]
+        case .jpeg: [0xFF, 0xD8]
+        }
+    }
+
+    var typeIdentifier: CFString {
+        switch self {
+        case .png: "public.png" as CFString
+        case .jpeg: "public.jpeg" as CFString
+        }
+    }
+
+    var destinationProperties: CFDictionary? {
+        switch self {
+        case .png:
+            // Lossless: `--quality` has no meaning here.
+            nil
+        case .jpeg(let quality):
+            [kCGImageDestinationLossyCompressionQuality: Double(quality) / 100.0] as CFDictionary
+        }
+    }
+}
+
 public struct VideoFrameUtilities {
     public static func makeCGImage(from data: Data) -> CGImage? {
         guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
@@ -69,13 +124,29 @@ public struct VideoFrameUtilities {
         return CGImageSourceCreateImageAtIndex(source, 0, nil)
     }
 
-    public static func processJPEGData(_ data: Data, scale: Double, quality: Int) async throws -> Data {
-        if scale < 1.0 {
-            return try await scaleJPEGData(data, scale: scale, quality: quality)
-        } else if quality != 80 {
-            return try await reencodeJPEGData(data, quality: quality)
+    /// Encode a freshly captured frame into `container`, applying `scale`
+    /// in the same pass. This is the one codec pass a frame from an
+    /// un-encoded source has to pay.
+    public static func encodeFrame(_ image: CGImage, as container: FrameContainer, scale: Double) throws -> Data {
+        let source = scale < 1.0 ? try resize(image, scale: scale) : image
+        return try encode(source, as: container)
+    }
+
+    /// Re-container an already-encoded frame, skipping the work when the
+    /// bytes are already what the sink needs. Only capture paths that
+    /// cannot choose their container need this — Android's `screencap`
+    /// emits PNG and nothing else.
+    public static func transcodeFrame(_ data: Data, to container: FrameContainer, scale: Double) throws -> Data {
+        // Passthrough is sound only for a lossless container: JPEG bytes
+        // carry no record of the quality they were encoded at, so
+        // honouring `--quality` means encoding them ourselves.
+        if scale >= 1.0, container == .png, data.starts(with: container.magicBytes) {
+            return data
+        }
+        guard let image = makeCGImage(from: data) else {
+            throw VideoProcessingError.failedToDecodeImage
         }
-        return data
+        return try encodeFrame(image, as: container, scale: scale)
     }
 
     public static func computeDimensions(for image: CGImage, scale: Double) -> (width: Int, height: Int) {
@@ -86,21 +157,18 @@ public struct VideoFrameUtilities {
         return (max(evenWidth, 2), max(evenHeight, 2))
     }
 
-    private static func encodeJPEG(_ image: CGImage, quality: Int) throws -> Data {
+    private static func encode(_ image: CGImage, as container: FrameContainer) throws -> Data {
         guard let data = CFDataCreateMutable(nil, 0),
               let destination = CGImageDestinationCreateWithData(
                   data,
-                  "public.jpeg" as CFString,
+                  container.typeIdentifier,
                   1,
                   nil
               ) else {
             throw VideoProcessingError.failedToEncodeImage
         }
 
-        let properties = [
-            kCGImageDestinationLossyCompressionQuality: Double(quality) / 100.0
-        ] as CFDictionary
-        CGImageDestinationAddImage(destination, image, properties)
+        CGImageDestinationAddImage(destination, image, container.destinationProperties)
 
         guard CGImageDestinationFinalize(destination) else {
             throw VideoProcessingError.failedToEncodeImage
@@ -109,11 +177,7 @@ public struct VideoFrameUtilities {
         return data as Data
     }
 
-    private static func scaleJPEGData(_ data: Data, scale: Double, quality: Int) async throws -> Data {
-        guard let image = makeCGImage(from: data) else {
-            throw VideoProcessingError.failedToDecodeImage
-        }
-
+    private static func resize(_ image: CGImage, scale: Double) throws -> CGImage {
         let dimensions = computeDimensions(for: image, scale: scale)
         guard let context = CGContext(
             data: nil,
@@ -142,16 +206,9 @@ public struct VideoFrameUtilities {
             throw VideoProcessingError.failedToEncodeImage
         }
 
-        return try encodeJPEG(scaledImage, quality: quality)
+        return scaledImage
     }
 
-    private static func reencodeJPEGData(_ data: Data, quality: Int) async throws -> Data {
-        guard let image = makeCGImage(from: data) else {
-            throw VideoProcessingError.failedToDecodeImage
-        }
-
-        return try encodeJPEG(image, quality: quality)
-    }
 }
 
 public final class H264StreamRecorder: Sendable {

Sources/iOSSimBackend/Util/VideoFrameUtilities+Simulator.swift

@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: Apache-2.0
+import CoreGraphics
 import Foundation
 import FBSimulatorControl
 @preconcurrency import FBControlCore
 import SimUseVideo
 
-// The FB*-tied capture entry point for the shared frame utilities.
+// The FB*-tied capture entry points for the shared frame utilities.
 // Everything else in `VideoFrameUtilities` is platform-neutral and lives
 // in SimUseVideo; this extension is the one piece that must stay inside
 // iOSSimBackend's FB* dep cone.
@@ -17,3 +18,33 @@ extension VideoFrameUtilities {
         return data
     }
 }
+
+/// A framebuffer-attached source of un-encoded frames.
+///
+/// `takeScreenshot` encodes on the simulator's behalf, which forces every
+/// consumer into whatever container it picked — PNG — and makes a stream
+/// that wants JPEG pay a decode plus a re-encode per frame. Reading the
+/// framebuffer directly hands out the `CGImage` instead, so each sink
+/// encodes exactly once into the container it needs, and the H.264
+/// recorder never encodes to an intermediate container at all.
+///
+/// Attaching a consumer to the framebuffer costs a surface handshake, so
+/// one source is created per capture session and reused for every frame.
+public final class SimulatorFrameSource {
+    private let image: FBSimulatorImage
+
+    public init(simulator: FBSimulator) async throws {
+        let framebuffer = try await simulator.connectToFramebuffer()
+        self.image = FBSimulatorImage.image(with: framebuffer, logger: simulator.logger)
+    }
+
+    /// The framebuffer's current contents. Throws rather than returning
+    /// nil so a frame loop can log and continue on a transient miss,
+    /// matching how it already handles capture failures.
+    public func currentFrame() throws -> CGImage {
+        guard let frame = image.image() else {
+            throw VideoProcessingError.emptyScreenshot
+        }
+        return frame
+    }
+}

Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift

@@ -215,10 +215,11 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand {
         scale: Double,
         cancellationFlag: CancellationFlag
     ) async throws {
-        let initialFrameData = try await VideoFrameUtilities.captureScreenshotData(from: simulator)
-        guard let initialImage = VideoFrameUtilities.makeCGImage(from: initialFrameData) else {
-            throw CLIError(errorDescription: "Failed to decode simulator screenshot")
-        }
+        // Frames come off the framebuffer un-encoded: the recorder wants a
+        // CGImage, so encoding to PNG only to decode it straight back was
+        // pure overhead on the one path that already has no fps headroom.
+        let frameSource = try await SimulatorFrameSource(simulator: simulator)
+        let initialImage = try frameSource.currentFrame()
 
         let dimensions = VideoFrameUtilities.computeDimensions(for: initialImage, scale: scale)
         let recorder = try H264StreamRecorder(
@@ -240,16 +241,14 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand {
             let frameStart = Date()
 
             do {
-                let frameData = try await VideoFrameUtilities.captureScreenshotData(from: simulator)
-                if let cgImage = VideoFrameUtilities.makeCGImage(from: frameData) {
-                    let now = Date()
-                    var presentationTime = CMTime(seconds: now.timeIntervalSince(writerStartTime), preferredTimescale: 600)
-                    if presentationTime <= lastPresentationTime {
-                        presentationTime = CMTimeAdd(lastPresentationTime, CMTime(value: 1, timescale: 600))
-                    }
-                    try recorder.append(image: cgImage, presentationTime: presentationTime)
-                    lastPresentationTime = presentationTime
+                let cgImage = try frameSource.currentFrame()
+                let now = Date()
+                var presentationTime = CMTime(seconds: now.timeIntervalSince(writerStartTime), preferredTimescale: 600)
+                if presentationTime <= lastPresentationTime {
+                    presentationTime = CMTimeAdd(lastPresentationTime, CMTime(value: 1, timescale: 600))
                 }
+                try recorder.append(image: cgImage, presentationTime: presentationTime)
+                lastPresentationTime = presentationTime
             } catch let error as VideoWriterStallError {
                 throw error
             } catch {