From d03f2205024df322da34b3e136a586bc985b6af5 Mon Sep 17 00:00:00 2001 From: "yuta.ooka" Date: Fri, 17 Jul 2026 16:06:30 +0900 Subject: [PATCH 1/2] feat: record-video captures real H.264 streams instead of screenshot polling Replace the screenshot-poll-and-re-encode recorder (capped ~8-10 fps) with native H.264 capture muxed straight into MP4 (passthrough, no re-encode): - iOS: FBSimulatorVideoStream eager H.264 at a constant --fps (1-60, default 30). The elementary stream carries no timestamps, so frames are laid out at exactly 1/fps -- the requested rate is honored and playback is smooth (uniform 16.7 ms spacing at 60 fps). - Android: adb screenrecord --output-format=h264 at the device's native variable rate, stitched across the API<34 180 s per-invocation limit. --fps is ignored there; --quality/--scale map to bitrate/size. New shared infra under Sources/iOSSimBackend/Util: AnnexBStreamParser (NAL splitting + access-unit assembly), H264PassthroughRecorder (AVAssetWriter passthrough; CFR for iOS, host-clock VFR for Android), H264MuxingPipeline; plus AdbStreamingProcess for incremental adb stdout. --fps range widened 1-30 -> 1-60 (default 10 -> 30). The legacy screenshot/screencap recorders are retained as automatic fallbacks. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 + README.md | 22 +- .../Adb/AdbStreamingProcess.swift | 136 ++++++++ Sources/SimUse/Commands/RecordVideo.swift | 226 +++++++++++-- .../Util/AnnexBStreamParser.swift | 256 +++++++++++++++ .../Util/H264MuxingPipeline.swift | 104 ++++++ .../Util/H264PassthroughRecorder.swift | 302 ++++++++++++++++++ .../Util/VideoCommandSupport.swift | 21 +- .../Verbs/IOSSimRecordVideoCommand.swift | 249 ++++++++++++--- Tests/AndroidRecordVideoArgumentTests.swift | 52 +++ Tests/AnnexBStreamParserTests.swift | 160 ++++++++++ Tests/Fixtures/sample-annexb-320x240.h264 | Bin 0 -> 11744 bytes Tests/Fixtures/sample-annexb.h264 | Bin 0 -> 7231 bytes Tests/H264MuxingPipelineTests.swift | 82 +++++ Tests/H264PassthroughRecorderTests.swift | 158 +++++++++ Tests/RecordVideoTests.swift | 46 ++- skills/sim-use/references/cheatsheet.md | 3 +- 17 files changed, 1734 insertions(+), 85 deletions(-) create mode 100644 Sources/AndroidBackend/Adb/AdbStreamingProcess.swift create mode 100644 Sources/iOSSimBackend/Util/AnnexBStreamParser.swift create mode 100644 Sources/iOSSimBackend/Util/H264MuxingPipeline.swift create mode 100644 Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift create mode 100644 Tests/AndroidRecordVideoArgumentTests.swift create mode 100644 Tests/AnnexBStreamParserTests.swift create mode 100644 Tests/Fixtures/sample-annexb-320x240.h264 create mode 100644 Tests/Fixtures/sample-annexb.h264 create mode 100644 Tests/H264MuxingPipelineTests.swift create mode 100644 Tests/H264PassthroughRecorderTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 928fb3a..d69c0ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `record-video` now captures a real H.264 stream and muxes it straight into the MP4 (passthrough) instead of polling screenshots and re-encoding each frame. iOS drives `FBSimulatorVideoStream` in eager H.264 mode at a constant `--fps` (frames laid out at exactly `1/fps`, so the rate is honored and playback is smooth); Android pipes `adb screenrecord --output-format=h264` at its native variable frame rate, stitching across `screenrecord` restarts past the API < 34 per-invocation limit. `--fps` is now `1–60` (default 30, was `1–30` default 10) and is ignored on Android; `--quality`/`--scale` map to bitrate/size on both platforms. A mid-recording rotation stops Android capture (an MP4 track can't change frame size). - The tap family (`tap`, `long-press`, `ios tap`) declares its shared flags once via `TapTargetingOptions` / `TapTimingOptions` option groups instead of three hand-kept copies (#42). Flag names, defaults, validation messages, and `--json` envelopes are unchanged; the only user-visible deltas are in `--help` output, where per-verb wording ("Tap the center…" / "Long-press the center…") unifies to verb-neutral phrasing ("Target the center…") and the timing flags render as a contiguous block after `--duration`. - `make e2e` now runs both the iOS and Android E2E suites in sequence (continuing past a platform failure and failing if either did); the iOS-only target is `make e2e-ios`, Android-only stays `make e2e-android`. - `scripts/test-runner.sh` keeps running after a failed suite and prints a full pass/fail map at the end (a release gate needs the whole picture, not the first crash). The hardcoded suite list gained the missing `KeyboardStateTests` and the new suites, and the misnamed `StreamVideoDebugTests` entry — which silently matched zero tests — now points at the real `StreamVideoDebugTest` suite. ### Fixed +- `record-video` frame rate is no longer capped at the ~8–10 fps the screenshot-polling loop topped out at (iOS honors `--fps` up to 60 with smooth constant-rate playback; an Android emulator captured ~50 fps under motion). Recording no longer burns CPU decoding/re-encoding every frame. - Four E2E suites (KeyTests, KeyComboTests, KeySequenceTests, StreamVideoTests) still invoked the pre-0.5.x top-level verb forms and had failed ever since the five iOS-only verbs moved under the `ios` namespace; they now call `sim-use ios `. - KeyComboTests' Cmd+A test asserted a cleared text field reads nil/empty — an empty `UITextField` exposes its placeholder as the accessibility value, so the assertion could never pass. It now accepts the placeholder form. diff --git a/README.md b/README.md index 863b727..4702aff 100644 --- a/README.md +++ b/README.md @@ -304,11 +304,27 @@ sim-use ios stream-video --device $UDID --fps 30 --format ffmpeg | \ ffmpeg -f image2pipe -framerate 30 -i - -c:v libx264 -preset ultrafast out.mp4 # Record MP4 directly (cross-platform) -sim-use record-video --device $UDID --fps 15 --output recording.mp4 -sim-use record-video --device $UDID --fps 10 --quality 60 --scale 0.5 --output low-bw.mp4 +sim-use record-video --device $UDID --output recording.mp4 # 30 fps default +sim-use record-video --device $UDID --fps 60 --output smooth.mp4 # up to 60 fps +sim-use record-video --device $UDID --quality 60 --scale 0.5 --output low-bw.mp4 ``` -Press Ctrl+C to stop; sim-use finalises the MP4 before exiting. +`record-video` captures a real H.264 stream and muxes it straight into the +MP4 (passthrough — no per-frame screenshot re-encoding): + + * **iOS** drives `FBSimulatorVideoStream` in eager H.264 mode at a constant + `--fps` (default 30, max 60). Because the stream carries no timestamps, + frames are laid out at exactly `1/fps`, so playback is smooth and the + requested rate is honored. + * **Android** pipes `adb screenrecord --output-format=h264` at the device's + native variable frame rate, so `--fps` is ignored there; `--quality` maps + to bitrate and `--scale` to `--size`. Recordings past the per-invocation + limit on API < 34 are stitched across `screenrecord` restarts + automatically. + +Rotating the display mid-recording stops capture on Android (an MP4 track +can't change frame size). Press Ctrl+C to stop; sim-use finalises the MP4 +before exiting. ### Accessibility inspection diff --git a/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift b/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift new file mode 100644 index 0000000..2dcaa6d --- /dev/null +++ b/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation + +/// A long-running `adb` child process whose stdout is delivered +/// incrementally as raw binary chunks — the streaming counterpart to +/// `Adb.run`, which buffers all output into a `String` and only returns +/// after exit. Used to pipe `adb exec-out screenrecord --output-format=h264` +/// straight into the H.264 muxer. +/// +/// Follows the same drain / termination-semaphore patterns as `Adb.run` +/// (readabilityHandler to avoid the 64 KB pipe deadlock, exit-driven wakeup, +/// ENOENT → `BridgeError.adbMissing`). +public final class AdbStreamingProcess: @unchecked Sendable { + private let adbPath: String + private let arguments: [String] + private let onStdout: @Sendable (Data) -> Void + private let onStderr: (@Sendable (String) -> Void)? + + private let process = Process() + private let stdoutPipe = Pipe() + private let stderrPipe = Pipe() + private let exitSemaphore = DispatchSemaphore(value: 0) + + private let lock = NSLock() + private var _stdoutByteCount: Int64 = 0 + private var stderrBuffer = Data() + + public init( + adbPath: String, + arguments: [String], + onStdout: @escaping @Sendable (Data) -> Void, + onStderr: (@Sendable (String) -> Void)? = nil + ) { + self.adbPath = adbPath + self.arguments = arguments + self.onStdout = onStdout + self.onStderr = onStderr + } + + public func start() throws { + let resolvedPath = Adb.resolveOnPATH(adbPath) ?? adbPath + process.executableURL = URL(fileURLWithPath: resolvedPath) + process.arguments = arguments + process.standardOutput = stdoutPipe + process.standardError = stderrPipe + + stdoutPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in + let chunk = handle.availableData + guard let self, !chunk.isEmpty else { return } + self.lock.lock() + self._stdoutByteCount += Int64(chunk.count) + self.lock.unlock() + self.onStdout(chunk) + } + stderrPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in + let chunk = handle.availableData + guard let self, !chunk.isEmpty else { return } + self.lock.lock() + self.stderrBuffer.append(chunk) + self.lock.unlock() + } + + process.terminationHandler = { [exitSemaphore] _ in exitSemaphore.signal() } + + do { + try process.run() + } catch { + let nsErr = error as NSError + let isMissing = + (nsErr.domain == NSCocoaErrorDomain && nsErr.code == 4) || + (nsErr.domain == NSPOSIXErrorDomain && nsErr.code == Int(ENOENT)) + if isMissing { + throw BridgeError.adbMissing + } + throw BridgeError.transport(underlying: "Failed to spawn adb: \(error.localizedDescription)", serial: nil) + } + } + + /// Send SIGINT — `screenrecord`'s clean-stop signal (flushes the encoder + /// and finalizes its output before exiting). + public func interrupt() { + guard process.isRunning else { return } + kill(process.processIdentifier, SIGINT) + } + + public func terminate() { + guard process.isRunning else { return } + process.terminate() + } + + public var isRunning: Bool { process.isRunning } + + public var stdoutByteCount: Int64 { + lock.lock(); defer { lock.unlock() } + return _stdoutByteCount + } + + public var collectedStderr: String { + lock.lock(); defer { lock.unlock() } + return String(data: stderrBuffer, encoding: .utf8) ?? "" + } + + /// Block until the child exits (or `timeout` elapses), then detach the + /// handlers and drain any residual stdout. Returns the exit status, or + /// nil on timeout (after a SIGTERM escalation). + @discardableResult + public func waitForExit(timeout: TimeInterval) -> Int32? { + let timedOut = exitSemaphore.wait(timeout: .now() + timeout) == .timedOut + if timedOut { + process.terminate() + _ = exitSemaphore.wait(timeout: .now() + 0.5) + } + + stdoutPipe.fileHandleForReading.readabilityHandler = nil + stderrPipe.fileHandleForReading.readabilityHandler = nil + + let residual = stdoutPipe.fileHandleForReading.readDataToEndOfFile() + if !residual.isEmpty { + lock.lock() + _stdoutByteCount += Int64(residual.count) + lock.unlock() + onStdout(residual) + } + let residualErr = stderrPipe.fileHandleForReading.readDataToEndOfFile() + if !residualErr.isEmpty { + lock.lock() + stderrBuffer.append(residualErr) + lock.unlock() + } + if let onStderr, !collectedStderr.isEmpty { + onStderr(collectedStderr) + } + + return timedOut ? nil : process.terminationStatus + } +} diff --git a/Sources/SimUse/Commands/RecordVideo.swift b/Sources/SimUse/Commands/RecordVideo.swift index bd1cc53..944acb6 100644 --- a/Sources/SimUse/Commands/RecordVideo.swift +++ b/Sources/SimUse/Commands/RecordVideo.swift @@ -11,18 +11,19 @@ import iOSSimBackend /// Top-level cross-platform `record-video` verb. Owns the flag /// surface and resolves the target platform, then delegates to: /// -/// * `IOSSimRecordVideoCommand.execute()` for iOS Simulator UDIDs. -/// * an inline Android orchestrator that drives `adb exec-out -/// screencap -p` and feeds the PNG frames into the same -/// `H264StreamRecorder` the iOS path uses. +/// * `IOSSimRecordVideoCommand.execute()` for iOS Simulator UDIDs +/// (which drives `FBSimulatorVideoStream` eager H.264 at `--fps`). +/// * an inline Android orchestrator that streams `adb exec-out +/// screenrecord --output-format=h264` into the shared H.264 → +/// MP4 passthrough muxer (`H264MuxingPipeline`). /// /// The Android branch lives inline (rather than in an /// `AndroidRecordVideoCommand` peer) because it cross-cuts -/// AndroidBackend (for `Adb`) and iOSSimBackend (for -/// `H264StreamRecorder` / `VideoFrameUtilities`). Only SimUse — the -/// executable target — depends on both modules, so this is the only -/// place where the orchestration can live without dragging -/// iOSSimBackend into AndroidBackend's dep cone. +/// AndroidBackend (for `Adb` / `AdbStreamingProcess`) and iOSSimBackend +/// (for the AVFoundation muxer). Only SimUse — the executable target — +/// depends on both modules, so this is the only place where the +/// orchestration can live without dragging iOSSimBackend into +/// AndroidBackend's dep cone. struct RecordVideo: SimUseExecutableCommand { typealias ExecutionResult = IOSSimRecordVideoCommand.ExecutionResult @@ -33,8 +34,8 @@ struct RecordVideo: SimUseExecutableCommand { @OptionGroup var device: DeviceOptions - @Option(help: "Frames per second (1-30, default: 10)") - var fps: Int = 10 + @Option(help: "Frames per second (1-60, default: 30). Ignored on Android (screenrecord uses the device's native variable frame rate).") + var fps: Int? @Option(help: "Quality factor (1-100) controlling bitrate (default: 80)") var quality: Int = 80 @@ -99,15 +100,22 @@ struct RecordVideo: SimUseExecutableCommand { // MARK: - Android - /// Android dispatch: drives a tight `adb exec-out screencap -p` - /// loop and feeds each PNG frame into the same - /// `H264StreamRecorder` used by the iOS path. Empirically caps - /// around 7–8 FPS on a typical emulator (PNG transfer dominates). + /// Raised only when `adb screenrecord` cannot produce an H.264 stream + /// (unsupported args, encoder unavailable). Triggers the legacy + /// screencap-frame fallback; mid-recording failures propagate as-is. + private struct ScreenrecordUnavailableError: Error { + let underlying: String + } + + /// Android dispatch: pipes `adb exec-out screenrecord + /// --output-format=h264 -` into the shared H.264 muxer for native, + /// variable-frame-rate capture. Falls back to the legacy + /// `screencap`-per-frame loop (≈7–8 FPS) only if screenrecord cannot + /// start. /// - /// Important: the bridge `/screenshot` path is NOT used here. - /// That route goes through `AccessibilityService.takeScreenshot` - /// which the Android framework rate-limits to ~2 FPS — unusable - /// for video. + /// The bridge `/screenshot` path is NOT used: it goes through + /// `AccessibilityService.takeScreenshot`, which the Android framework + /// rate-limits to ~2 FPS — unusable for video. private func executeAndroid() async throws -> ExecutionResult { let adb = Adb() let serial = device.resolved @@ -126,7 +134,7 @@ struct RecordVideo: SimUseExecutableCommand { defer { signalObserver.invalidate() } do { - try await recordVideoAndroid( + try await recordVideoAndroidStream( adb: adb, serial: serial, outputURL: outputURL, @@ -137,6 +145,24 @@ struct RecordVideo: SimUseExecutableCommand { ) recordingFinished.cancel() return ExecutionResult(path: outputURL.path) + } catch let unavailable as ScreenrecordUnavailableError { + FileHandle.standardError.write(Data("warning: screenrecord unavailable (\(unavailable.underlying)); falling back to screencap frames\n".utf8)) + do { + try await recordVideoAndroidScreencapLegacy( + adb: adb, + serial: serial, + outputURL: outputURL, + fps: fps ?? 10, + quality: quality, + scale: scale, + cancellationFlag: cancellationFlag + ) + recordingFinished.cancel() + return ExecutionResult(path: outputURL.path) + } catch { + recordingFinished.cancel() + throw CLIError(errorDescription: "Failed to record video: \(error.localizedDescription)") + } } catch { recordingFinished.cancel() throw CLIError(errorDescription: "Failed to record video: \(error.localizedDescription)") @@ -158,7 +184,165 @@ struct RecordVideo: SimUseExecutableCommand { } } - private func recordVideoAndroid( + /// Native capture: `adb exec-out screenrecord --output-format=h264 -` + /// streamed into the shared muxer. On API < 34 `screenrecord` self-limits + /// to 180 s per invocation, so we restart it in a loop and keep feeding + /// the same muxer — the single host clock keeps PTS continuous across the + /// ~100–300 ms restart gap. + private func recordVideoAndroidStream( + adb: Adb, + serial: String, + outputURL: URL, + fps: Int?, + quality: Int, + scale: Double, + cancellationFlag: CancellationFlag + ) async throws { + if fps != nil { + FileHandle.standardError.write(Data("note: --fps is ignored on Android (screenrecord records at native variable frame rate)\n".utf8)) + } + + let sdk = Self.detectSDK(adb: adb, serial: serial) + let size = scale < 1.0 ? Self.detectScaledSize(adb: adb, serial: serial, scale: scale) : nil + let bitrate = size.map { H264StreamRecorder.estimateBitrate(width: $0.width, height: $0.height, fps: 30, quality: quality) } + let arguments = Self.screenrecordArguments(serial: serial, sdk: sdk, bitrate: bitrate, size: size) + + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + var recorderFinalized = false + defer { if !recorderFinalized { recorder.invalidate() } } + + let fatalBox = FirstErrorBox() + let pipeline = H264MuxingPipeline(recorder: recorder, onFatalError: { error in + fatalBox.set(error) + cancellationFlag.cancel() + }) + + var firstSegment = true + var disconnected = false + + segmentLoop: while true { + if Task.isCancelled || cancellationFlag.isCancelled() || fatalBox.first != nil { break } + + pipeline.resetParserForNewSegment() + let process = AdbStreamingProcess( + adbPath: adb.binaryPath, + arguments: arguments, + onStdout: { pipeline.ingest($0) } + ) + do { + try process.start() + } catch { + if firstSegment { + throw ScreenrecordUnavailableError(underlying: error.localizedDescription) + } + throw error + } + firstSegment = false + + let segmentStartBytes = process.stdoutByteCount + while process.isRunning { + if Task.isCancelled || cancellationFlag.isCancelled() || fatalBox.first != nil { break } + try? await cancellableSleep(seconds: 0.05, flag: cancellationFlag) + } + + let stopping = Task.isCancelled || cancellationFlag.isCancelled() || fatalBox.first != nil + if stopping { + process.interrupt() + process.waitForExit(timeout: 2) + break + } + + // The process exited on its own — either the API-level time limit + // was reached (restart to continue) or the device stopped feeding. + let exitCode = process.waitForExit(timeout: 2) + let bytesThisSegment = process.stdoutByteCount - segmentStartBytes + if bytesThisSegment == 0 { + if !pipeline.firstFrameReceived { + let exitDescription = exitCode.map(String.init) ?? "timeout" + throw ScreenrecordUnavailableError( + underlying: "screenrecord produced no output (exit \(exitDescription)): \(process.collectedStderr.trimmingCharacters(in: .whitespacesAndNewlines))" + ) + } + disconnected = true + break segmentLoop + } + FileHandle.standardError.write(Data("screenrecord segment ended (Android time limit); restarting (~100-300ms gap)\n".utf8)) + } + + pipeline.finishIngest() + do { + try await recorder.finish(stopHostTime: ProcessInfo.processInfo.systemUptime) + recorderFinalized = true + } catch { + if let fatal = fatalBox.first { throw fatal } + throw error + } + + if let fatal = fatalBox.first { throw fatal } + if disconnected { + throw CLIError(errorDescription: "Android device stopped producing frames during recording; partial recording saved to \(outputURL.path)") + } + } + + private static func detectSDK(adb: Adb, serial: String) -> Int { + guard let result = try? adb.shell(serial: serial, args: ["getprop", "ro.build.version.sdk"]) else { + return 30 + } + return Int(result.stdout.trimmingCharacters(in: .whitespacesAndNewlines)) ?? 30 + } + + private static func detectScaledSize(adb: Adb, serial: String, scale: Double) -> (width: Int, height: Int)? { + guard let result = try? adb.shell(serial: serial, args: ["wm", "size"]), + let base = parseWMSize(result.stdout) else { + return nil + } + let width = max(2, Int(Double(base.width) * scale)) + let height = max(2, Int(Double(base.height) * scale)) + return (width - (width % 2), height - (height % 2)) + } + + /// Parse `adb shell wm size` output. Prefers the `Override size:` line + /// (an active resolution override) over `Physical size:`. + static func parseWMSize(_ output: String) -> (width: Int, height: Int)? { + func size(from line: Substring) -> (Int, Int)? { + guard let colon = line.lastIndex(of: ":") else { return nil } + let value = line[line.index(after: colon)...].trimmingCharacters(in: .whitespaces) + let parts = value.split(separator: "x") + guard parts.count == 2, let w = Int(parts[0]), let h = Int(parts[1]) else { return nil } + return (w, h) + } + let lines = output.split(separator: "\n") + if let override = lines.first(where: { $0.contains("Override size:") }), let parsed = size(from: override) { + return parsed + } + if let physical = lines.first(where: { $0.contains("Physical size:") }), let parsed = size(from: physical) { + return parsed + } + return nil + } + + /// Build the `adb screenrecord` argument vector. `--time-limit 0` + /// (unlimited) is only valid on API ≥ 34; older devices hard-cap at 180 s, + /// which the segment loop handles by restarting. + static func screenrecordArguments(serial: String, sdk: Int, bitrate: Int?, size: (width: Int, height: Int)?) -> [String] { + var arguments = ["-s", serial, "exec-out", "screenrecord", "--output-format=h264"] + if sdk >= 34 { + arguments.append(contentsOf: ["--time-limit", "0"]) + } + if let bitrate { + arguments.append(contentsOf: ["--bit-rate", "\(bitrate)"]) + } + if let size { + arguments.append(contentsOf: ["--size", "\(size.width)x\(size.height)"]) + } + arguments.append("-") + return arguments + } + + /// Legacy screencap-per-frame recorder, retained as an automatic fallback + /// for when `screenrecord --output-format=h264` is unavailable. Caps + /// around 7–8 FPS on a typical emulator (PNG transfer dominates). + private func recordVideoAndroidScreencapLegacy( adb: Adb, serial: String, outputURL: URL, diff --git a/Sources/iOSSimBackend/Util/AnnexBStreamParser.swift b/Sources/iOSSimBackend/Util/AnnexBStreamParser.swift new file mode 100644 index 0000000..790d474 --- /dev/null +++ b/Sources/iOSSimBackend/Util/AnnexBStreamParser.swift @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation + +/// A single H.264 NAL unit: the 1-byte header plus its EBSP payload, +/// with the Annex-B start code already stripped. +public struct H264NALUnit: Equatable { + /// `nal_unit_type` — the low 5 bits of the header byte. + public let type: UInt8 + /// Header byte + payload, no start code, emulation-prevention bytes intact. + public let data: Data + + public init(type: UInt8, data: Data) { + self.type = type + self.data = data + } + + /// VCL NAL unit types (1...5) carry coded slice data; everything else + /// (SPS/PPS/SEI/AUD/…) is non-VCL. + public var isVCL: Bool { type >= 1 && type <= 5 } +} + +/// One access unit (a single coded picture) in decode order. SPS/PPS and +/// AUD NAL units are stripped out during assembly — parameter sets go into +/// the mp4 `avcC` box, and AUDs are only boundary hints. +public struct H264AccessUnit: Equatable { + public let nalUnits: [H264NALUnit] + /// True when the AU contains an IDR slice (NAL type 5) — the mp4 muxer + /// marks these as sync samples. + public let isIDR: Bool + + public init(nalUnits: [H264NALUnit], isIDR: Bool) { + self.nalUnits = nalUnits + self.isIDR = isIDR + } +} + +/// Incremental Annex-B byte-stream parser. Feed arbitrary chunks (as they +/// arrive from `FBSimulatorVideoStream` or `adb screenrecord`) and receive +/// complete access units. Start codes may straddle chunk boundaries; a NAL +/// unit is only emitted once the *next* start code confirms its end. +public final class AnnexBStreamParser { + /// Latest SPS (NAL type 7), stripped of its start code. + public private(set) var currentSPS: Data? + /// Latest PPS (NAL type 8), stripped of its start code. + public private(set) var currentPPS: Data? + + /// Bytes seen but not yet split into a complete NAL unit. Always begins + /// at the most recent start code (or is empty), so it never outgrows one + /// in-flight NAL unit. + private var buffer = Data() + + /// NAL units accumulated for the access unit currently being assembled. + private var pending: [H264NALUnit] = [] + private var pendingHasVCL = false + private var pendingIsIDR = false + + /// Access units completed during the in-progress `consume(_:)` call. + private var completed: [H264AccessUnit] = [] + + public init() {} + + /// Feed a chunk; returns every access unit that became complete as a + /// result (possibly none). + public func consume(_ data: Data) -> [H264AccessUnit] { + buffer.append(data) + for nalu in extractNALUnits() { + route(nalu) + } + defer { completed.removeAll(keepingCapacity: true) } + return completed + } + + /// End-of-stream flush: drain the final NAL unit (which has no following + /// start code to terminate it) and close the trailing access unit. This + /// can emit up to two units — draining the last NAL unit may first close + /// the previous picture before the final one is closed. Safe to call more + /// than once (returns empty after the first). + public func flush() -> [H264AccessUnit] { + for nalu in drainFinalNALUnit() { + route(nalu) + } + closePendingAU() + defer { completed.removeAll(keepingCapacity: true) } + return completed + } + + // MARK: - NAL unit routing / AU assembly + + /// H.264 access-unit boundary rule (simplified for non-reordered, + /// in-order streams from our two capture sources): a new AU begins at + /// the first VCL slice with `first_mb_in_slice == 0`, or at any leading + /// non-VCL unit (AUD/SPS/PPS/SEI) that follows a VCL unit. + private func route(_ nalu: H264NALUnit) { + if nalu.isVCL { + let firstMB = Self.firstMBInSlice(nalu.data) ?? 0 + if firstMB == 0 && pendingHasVCL { + closePendingAU() + } + pending.append(nalu) + pendingHasVCL = true + if nalu.type == 5 { pendingIsIDR = true } + return + } + + // Non-VCL unit after a VCL means the previous picture is done. + if pendingHasVCL { + closePendingAU() + } + switch nalu.type { + case 7: currentSPS = nalu.data + case 8: currentPPS = nalu.data + case 9: break // AUD: boundary hint only, not carried into the AU + default: pending.append(nalu) // SEI and friends lead the next AU + } + } + + private func closePendingAU() { + defer { + pending.removeAll(keepingCapacity: true) + pendingHasVCL = false + pendingIsIDR = false + } + guard pendingHasVCL else { return } + completed.append(H264AccessUnit(nalUnits: pending, isIDR: pendingIsIDR)) + } + + // MARK: - Start-code splitting + + /// Split `buffer` into complete NAL units, leaving the trailing + /// (not-yet-terminated) unit in `buffer` for the next chunk. + private func extractNALUnits() -> [H264NALUnit] { + let bytes = [UInt8](buffer) + let starts = Self.startCodeOffsets(bytes) + guard !starts.isEmpty else { return [] } + + var nalus: [H264NALUnit] = [] + for index in starts.indices { + let payloadStart = starts[index] + 3 + // The last start code has no terminator yet — keep it buffered. + guard index + 1 < starts.count else { break } + if let nalu = Self.makeNALUnit(bytes, payloadStart: payloadStart, rawEnd: starts[index + 1]) { + nalus.append(nalu) + } + } + + buffer = Data(bytes[starts[starts.count - 1]...]) + return nalus + } + + /// EOF variant: the final start code's NAL unit runs to the end of the + /// buffer with no terminating start code. + private func drainFinalNALUnit() -> [H264NALUnit] { + let bytes = [UInt8](buffer) + let starts = Self.startCodeOffsets(bytes) + guard let last = starts.last else { return [] } + buffer.removeAll(keepingCapacity: false) + guard let nalu = Self.makeNALUnit(bytes, payloadStart: last + 3, rawEnd: bytes.count) else { + return [] + } + return [nalu] + } + + /// Offsets of every 3-byte `00 00 01` start-code prefix. A 4-byte + /// `00 00 00 01` is found as its trailing 3 bytes; the extra leading + /// zero is trimmed as a trailing byte of the preceding unit. Emulation + /// prevention guarantees `00 00 01` never appears inside a NAL payload. + static func startCodeOffsets(_ bytes: [UInt8]) -> [Int] { + var offsets: [Int] = [] + guard bytes.count >= 3 else { return offsets } + var i = 0 + while i <= bytes.count - 3 { + if bytes[i] == 0 && bytes[i + 1] == 0 && bytes[i + 2] == 1 { + offsets.append(i) + i += 3 + } else { + i += 1 + } + } + return offsets + } + + private static func makeNALUnit(_ bytes: [UInt8], payloadStart: Int, rawEnd: Int) -> H264NALUnit? { + var end = rawEnd + // Trim trailing zeros: they belong to the next start code + // (`00 00 00 01`) or are cabac_zero_word / trailing_zero_8bits. + while end > payloadStart && bytes[end - 1] == 0 { end -= 1 } + guard end > payloadStart else { return nil } + let slice = bytes[payloadStart.. UInt? { + guard nalu.count > 1 else { return nil } + let rbsp = deEscape(Array(nalu.dropFirst()), maxBytes: 8) + var reader = BitReader(rbsp) + return reader.readUnsignedExpGolomb() + } + + /// Remove emulation-prevention bytes (`00 00 03` → `00 00`) from the + /// leading `maxBytes` of an EBSP so the slice header can be bit-read. + static func deEscape(_ bytes: [UInt8], maxBytes: Int) -> [UInt8] { + var out: [UInt8] = [] + out.reserveCapacity(min(bytes.count, maxBytes)) + var zeroRun = 0 + for byte in bytes { + if zeroRun >= 2 && byte == 0x03 { + zeroRun = 0 + continue + } + out.append(byte) + zeroRun = (byte == 0) ? zeroRun + 1 : 0 + if out.count >= maxBytes { break } + } + return out + } +} + +/// Minimal MSB-first bit reader over a de-escaped RBSP byte array. +struct BitReader { + private let bytes: [UInt8] + private var bitPos = 0 + + init(_ bytes: [UInt8]) { self.bytes = bytes } + + mutating func readBit() -> UInt8? { + let byteIndex = bitPos >> 3 + guard byteIndex < bytes.count else { return nil } + let bit = (bytes[byteIndex] >> (7 - (bitPos & 7))) & 1 + bitPos += 1 + return bit + } + + /// Decode an unsigned Exp-Golomb `ue(v)` code. + mutating func readUnsignedExpGolomb() -> UInt? { + var leadingZeros = 0 + while true { + guard let bit = readBit() else { return nil } + if bit == 1 { break } + leadingZeros += 1 + if leadingZeros > 31 { return nil } + } + guard leadingZeros > 0 else { return 0 } + var suffix: UInt = 0 + for _ in 0.. TimeInterval + private let onFatalError: @Sendable (Error) -> Void + + private let lock = NSLock() + private var parser = AnnexBStreamParser() + private var fatal = false + private var closed = false + private var _framesWritten: Int64 = 0 + private var _firstFrameReceived = false + private var _lastIngestHostTime: TimeInterval = 0 + + public init( + recorder: H264PassthroughRecorder, + clock: @escaping @Sendable () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, + onFatalError: @escaping @Sendable (Error) -> Void + ) { + self.recorder = recorder + self.clock = clock + self.onFatalError = onFatalError + } + + /// Parse a chunk and append any completed access units. No-op once the + /// pipeline has been closed by `finishIngest()` or latched by an error. + public func ingest(_ data: Data) { + lock.lock() + defer { lock.unlock() } + guard !fatal, !closed else { return } + let now = clock() + _lastIngestHostTime = now + appendLocked(parser.consume(data), hostTime: now) + } + + /// Start a fresh parser for a new capture segment (Android restarts + /// `screenrecord` every 180 s on API < 34). The single monotonic clock + /// keeps PTS continuous across the boundary. + public func resetParserForNewSegment() { + lock.lock() + defer { lock.unlock() } + parser = AnnexBStreamParser() + } + + /// Flush the parser's trailing access unit and close the intake. After + /// this returns, no further `ingest(_:)` will append, so the caller can + /// safely finalize the recorder. Idempotent. + public func finishIngest() { + lock.lock() + defer { lock.unlock() } + guard !fatal, !closed else { return } + closed = true + // Stamp the trailing frame with when its data actually arrived, not + // now — the parser holds a frame pending until the next start code, so + // a static screen's sole keyframe arrives at t≈0 but is only flushed + // at stop. Using the flush time would pin the first frame's PTS to the + // stop instant and collapse the clip to zero duration. + let hostTime = _lastIngestHostTime > 0 ? _lastIngestHostTime : clock() + appendLocked(parser.flush(), hostTime: hostTime) + } + + private func appendLocked(_ accessUnits: [H264AccessUnit], hostTime: TimeInterval) { + guard !accessUnits.isEmpty, let sps = parser.currentSPS, let pps = parser.currentPPS else { return } + do { + for accessUnit in accessUnits { + try recorder.append(accessUnit: accessUnit, sps: sps, pps: pps, hostTime: hostTime) + _framesWritten += 1 + _firstFrameReceived = true + } + } catch { + fatal = true + onFatalError(error) + } + } + + public var framesWritten: Int64 { + lock.lock(); defer { lock.unlock() } + return _framesWritten + } + + public var firstFrameReceived: Bool { + lock.lock(); defer { lock.unlock() } + return _firstFrameReceived + } + + public var lastIngestHostTime: TimeInterval { + lock.lock(); defer { lock.unlock() } + return _lastIngestHostTime + } + + public var hadFatalError: Bool { + lock.lock(); defer { lock.unlock() } + return fatal + } +} diff --git a/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift b/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift new file mode 100644 index 0000000..fe0b430 --- /dev/null +++ b/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import AVFoundation +import CoreMedia +import SimUseCore + +public enum H264PassthroughError: Error, LocalizedError, Equatable { + /// The recording was finalized without a single decodable frame. + case noFramesCaptured + /// SPS/PPS could not be turned into a CMFormatDescription. + case missingParameterSets + /// A later stream segment reported different display dimensions + /// (e.g. a rotation) than the first — passthrough can't splice these. + case dimensionsChanged(old: String, new: String) + case appendFailed(String) + + public var errorDescription: String? { + switch self { + case .noFramesCaptured: + return "No video frames were captured (static screen or the stream never produced a keyframe)." + case .missingParameterSets: + return "H.264 parameter sets (SPS/PPS) were missing or malformed." + case let .dimensionsChanged(old, new): + return "Display size changed mid-recording (\(old) → \(new)); recording stopped." + case let .appendFailed(detail): + return "Failed to append video sample: \(detail)" + } + } +} + +/// Muxes an already-encoded H.264 elementary stream into an MP4 without +/// re-encoding. Access units (from `AnnexBStreamParser`) are rewrapped as +/// AVCC-framed `CMSampleBuffer`s and appended to a passthrough +/// `AVAssetWriterInput`. Presentation timestamps come from the host arrival +/// clock, since neither capture source embeds timing. +public final class H264PassthroughRecorder: @unchecked Sendable { + private let outputURL: URL + private let writer: AVAssetWriter + /// When set, timestamps are laid out as a constant frame rate + /// (`frameIndex / frameRate`) instead of derived from host arrival time. + /// This removes the byte-arrival jitter of an eager fixed-rate stream; + /// nil keeps the variable-rate host-clock behavior (Android screenrecord). + private let frameRate: Int? + private var input: AVAssetWriterInput? + + private var formatDescription: CMFormatDescription? + private var formatSPS: Data? + private var formatPPS: Data? + private var formatDimensions: CMVideoDimensions? + + private var firstHostTime: TimeInterval? + private var lastPTS: CMTime = .invalid + + public private(set) var framesAppended: Int64 = 0 + + public init(outputURL: URL, frameRate: Int? = nil) throws { + self.outputURL = outputURL + self.frameRate = frameRate + self.writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + } + + /// Append one access unit. The first call lazily builds the format + /// description and starts the writer session (parameter sets are only + /// known once the first SPS/PPS arrive). + public func append(accessUnit: H264AccessUnit, sps: Data, pps: Data, hostTime: TimeInterval) throws { + try ensureStarted(sps: sps, pps: pps, hostTime: hostTime) + guard let input, let formatDescription, let firstHostTime else { + throw H264PassthroughError.appendFailed("writer not initialized") + } + + try H264StreamRecorder.waitUntilReady( + isReady: { input.isReadyForMoreMediaData }, + timeout: H264StreamRecorder.readinessTimeout + ) + + let pts: CMTime + if let frameRate { + pts = CMTime(value: framesAppended, timescale: CMTimeScale(frameRate)) + } else { + pts = Self.normalizedPTS(hostTime: hostTime, firstHostTime: firstHostTime, lastPTS: lastPTS) + } + let sampleBuffer = try Self.makeSampleBuffer( + avcc: Self.avccData(for: accessUnit), + formatDescription: formatDescription, + pts: pts, + isIDR: accessUnit.isIDR + ) + guard input.append(sampleBuffer) else { + throw H264PassthroughError.appendFailed(writer.error?.localizedDescription ?? "unknown writer error") + } + lastPTS = pts + framesAppended += 1 + } + + /// Finalize the MP4. `stopHostTime` sets the session end so the final + /// (variable-rate) frame is held for its true wall-clock duration. + /// Throws `.noFramesCaptured` — after deleting the empty output — when + /// no frame was ever appended. + public func finish(stopHostTime: TimeInterval?) async throws { + guard let input, framesAppended > 0, let firstHostTime else { + if writer.status == .writing { + writer.cancelWriting() + } + try? FileManager.default.removeItem(at: outputURL) + throw H264PassthroughError.noFramesCaptured + } + + if let frameRate { + // Hold the last frame for one frame interval past its PTS. + writer.endSession(atSourceTime: CMTime(value: framesAppended, timescale: CMTimeScale(frameRate))) + } else if let stopHostTime { + let endPTS = Self.normalizedPTS(hostTime: stopHostTime, firstHostTime: firstHostTime, lastPTS: lastPTS) + writer.endSession(atSourceTime: endPTS) + } + input.markAsFinished() + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + writer.finishWriting { + if let error = self.writer.error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: ()) + } + } + } + } + + public func invalidate() { + if writer.status == .writing { + input?.markAsFinished() + writer.cancelWriting() + } + } + + // MARK: - Startup / segment handling + + private func ensureStarted(sps: Data, pps: Data, hostTime: TimeInterval) throws { + guard input == nil else { + // Already recording: only react if the parameter sets changed + // (new segment on Android's 180 s restart, or a rotation). + guard sps != formatSPS || pps != formatPPS else { return } + let newFormat = try Self.makeFormatDescription(sps: sps, pps: pps) + let newDimensions = CMVideoFormatDescriptionGetDimensions(newFormat) + if let old = formatDimensions, old.width != newDimensions.width || old.height != newDimensions.height { + throw H264PassthroughError.dimensionsChanged( + old: "\(old.width)x\(old.height)", + new: "\(newDimensions.width)x\(newDimensions.height)" + ) + } + FileHandle.standardError.write(Data("note: stream parameter sets changed mid-recording (same size); continuing\n".utf8)) + formatDescription = newFormat + formatSPS = sps + formatPPS = pps + return + } + + let format = try Self.makeFormatDescription(sps: sps, pps: pps) + let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: nil, sourceFormatHint: format) + writerInput.expectsMediaDataInRealTime = true + guard writer.canAdd(writerInput) else { + throw H264PassthroughError.appendFailed("writer cannot add passthrough input") + } + writer.add(writerInput) + guard writer.startWriting() else { + throw H264PassthroughError.appendFailed(writer.error?.localizedDescription ?? "startWriting failed") + } + writer.startSession(atSourceTime: .zero) + + input = writerInput + formatDescription = format + formatSPS = sps + formatPPS = pps + formatDimensions = CMVideoFormatDescriptionGetDimensions(format) + firstHostTime = hostTime + } + + // MARK: - Pure helpers (unit-tested) + + /// Concatenate an access unit's NAL units in AVCC framing: each prefixed + /// with its 4-byte big-endian length (matching `nalUnitHeaderLength: 4`). + static func avccData(for accessUnit: H264AccessUnit) -> Data { + var out = Data() + for nalu in accessUnit.nalUnits { + var length = UInt32(nalu.data.count).bigEndian + withUnsafeBytes(of: &length) { out.append(contentsOf: $0) } + out.append(nalu.data) + } + return out + } + + /// Map a host arrival time to a monotonic PTS on a 600 timescale. The + /// first frame is pinned to zero (the session start); any non-increasing + /// value is bumped one tick past the previous PTS, since passthrough + /// inputs reject out-of-order timestamps. + static func normalizedPTS(hostTime: TimeInterval, firstHostTime: TimeInterval, lastPTS: CMTime) -> CMTime { + let seconds = max(0, hostTime - firstHostTime) + var pts = CMTime(seconds: seconds, preferredTimescale: 600) + if lastPTS.isValid, pts <= lastPTS { + pts = CMTimeAdd(lastPTS, CMTime(value: 1, timescale: 600)) + } + return pts + } + + static func makeFormatDescription(sps: Data, pps: Data) throws -> CMFormatDescription { + guard !sps.isEmpty, !pps.isEmpty else { + throw H264PassthroughError.missingParameterSets + } + var format: CMFormatDescription? + let status = sps.withUnsafeBytes { spsRaw in + pps.withUnsafeBytes { ppsRaw -> OSStatus in + guard + let spsBase = spsRaw.bindMemory(to: UInt8.self).baseAddress, + let ppsBase = ppsRaw.bindMemory(to: UInt8.self).baseAddress + else { return -1 } + let pointers = [spsBase, ppsBase] + let sizes = [sps.count, pps.count] + return pointers.withUnsafeBufferPointer { pointerBuffer in + sizes.withUnsafeBufferPointer { sizeBuffer in + CMVideoFormatDescriptionCreateFromH264ParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: 2, + parameterSetPointers: pointerBuffer.baseAddress!, + parameterSetSizes: sizeBuffer.baseAddress!, + nalUnitHeaderLength: 4, + formatDescriptionOut: &format + ) + } + } + } + } + guard status == noErr, let format else { + throw H264PassthroughError.missingParameterSets + } + return format + } + + private static func makeSampleBuffer( + avcc: Data, + formatDescription: CMFormatDescription, + pts: CMTime, + isIDR: Bool + ) throws -> CMSampleBuffer { + var blockBuffer: CMBlockBuffer? + let blockStatus = CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: avcc.count, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: avcc.count, + flags: 0, + blockBufferOut: &blockBuffer + ) + guard blockStatus == kCMBlockBufferNoErr, let blockBuffer else { + throw H264PassthroughError.appendFailed("CMBlockBuffer create failed (\(blockStatus))") + } + let copyStatus = avcc.withUnsafeBytes { raw in + CMBlockBufferReplaceDataBytes( + with: raw.baseAddress!, + blockBuffer: blockBuffer, + offsetIntoDestination: 0, + dataLength: avcc.count + ) + } + guard copyStatus == kCMBlockBufferNoErr else { + throw H264PassthroughError.appendFailed("CMBlockBuffer copy failed (\(copyStatus))") + } + + var sampleBuffer: CMSampleBuffer? + var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: pts, decodeTimeStamp: .invalid) + var sampleSize = avcc.count + let sampleStatus = CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, + dataBuffer: blockBuffer, + formatDescription: formatDescription, + sampleCount: 1, + sampleTimingEntryCount: 1, + sampleTimingArray: &timing, + sampleSizeEntryCount: 1, + sampleSizeArray: &sampleSize, + sampleBufferOut: &sampleBuffer + ) + guard sampleStatus == noErr, let sampleBuffer else { + throw H264PassthroughError.appendFailed("CMSampleBuffer create failed (\(sampleStatus))") + } + + // A sample is a sync sample unless flagged otherwise; only non-IDR + // pictures need the NotSync attachment so seeking lands on keyframes. + if !isIDR, + let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: true), + CFArrayGetCount(attachments) > 0 { + let dictionary = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue( + dictionary, + Unmanaged.passUnretained(kCMSampleAttachmentKey_NotSync).toOpaque(), + Unmanaged.passUnretained(kCFBooleanTrue).toOpaque() + ) + } + + return sampleBuffer + } +} diff --git a/Sources/iOSSimBackend/Util/VideoCommandSupport.swift b/Sources/iOSSimBackend/Util/VideoCommandSupport.swift index 442ff81..86ca1f0 100644 --- a/Sources/iOSSimBackend/Util/VideoCommandSupport.swift +++ b/Sources/iOSSimBackend/Util/VideoCommandSupport.swift @@ -36,6 +36,25 @@ public final class CancellationFlag: @unchecked Sendable { } } +/// A latch that transitions to "set" exactly once. Guarantees a +/// `CheckedContinuation` is resumed a single time when two callbacks race +/// (a completion handler versus a timeout). +public final class OnceFlag: @unchecked Sendable { + private let lock = NSLock() + private var fired = false + + public init() {} + + /// Returns true the first time it is called, false thereafter. + public func trySet() -> Bool { + lock.lock() + defer { lock.unlock() } + guard !fired else { return false } + fired = true + return true + } +} + /// Thread-safe, set-once error capture for callback-based FBFutures. /// /// The BGRA stream reports failures through `FBFuture` completion @@ -403,7 +422,7 @@ public final class H264StreamRecorder: @unchecked Sendable { return pixelBuffer } - private static func estimateBitrate(width: Int, height: Int, fps: Int, quality: Int) -> Int { + public static func estimateBitrate(width: Int, height: Int, fps: Int, quality: Int) -> Int { let qualityFactor = max(0.1, min(Double(quality) / 100.0, 1.0)) let bitsPerPixel = 0.1 + (0.4 * qualityFactor) let bitrate = Double(width * height) * bitsPerPixel * Double(fps) diff --git a/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift b/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift index ae994e4..7d23c03 100644 --- a/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift +++ b/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift @@ -6,14 +6,17 @@ import FBSimulatorControl import AVFoundation import SimUseCore -/// iOS Simulator backend for the `record-video` verb. The top-level -/// cross-platform `record-video` forwards iOS UDIDs through here; the -/// Android branch keeps its execute body inline in the forwarder -/// because it cross-cuts AndroidBackend (for `Adb`) and iOSSimBackend -/// (for `H264StreamRecorder` / `VideoFrameUtilities`). Only SimUse — -/// the executable target — depends on both, so AndroidBackend cannot -/// host the Android record-video orchestrator without dragging -/// iOSSimBackend into its dep cone. +/// iOS Simulator backend for the `record-video` verb. Recording drives +/// `FBSimulatorVideoStream` in eager (fixed-rate) H.264 mode and muxes the +/// resulting Annex-B elementary stream straight into an MP4 (passthrough, no +/// re-encode). Because the stream carries no timestamps, presentation times +/// are laid out as a constant frame rate (`--fps`) — this is what makes the +/// requested frame rate honorable and keeps playback smooth (an eager +/// stream's bytes arrive in bursts, so deriving PTS from arrival time would +/// judder). +/// +/// idb's lazy (damage-driven) mode is unused: it emits no frames on the +/// modern Metal-backed simulator surface even during motion. public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { public static let configuration = CommandConfiguration( commandName: "record-video", @@ -27,10 +30,17 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { } } + /// Raised only when the H.264 video stream cannot be set up (private + /// CoreSimulator API unavailable, stream fails to start). Triggers the + /// screenshot-capture fallback; mid-recording failures propagate as-is. + private struct StreamUnavailableError: Error { + let underlying: String + } + @OptionGroup public var device: DeviceOptions - @Option(help: "Frames per second (1-30, default: 10)") - public var fps: Int = 10 + @Option(help: "Frames per second (1-60, default: 30).") + public var fps: Int? @Option(help: "Quality factor (1-100) controlling bitrate (default: 80)") public var quality: Int = 80 @@ -66,9 +76,11 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { try Self.validateOptions(fps: fps, quality: quality, scale: scale) } - public static func validateOptions(fps: Int, quality: Int, scale: Double) throws { - guard fps >= 1 && fps <= 30 else { - throw ValidationError("FPS must be between 1 and 30") + public static func validateOptions(fps: Int?, quality: Int, scale: Double) throws { + if let fps { + guard fps >= 1 && fps <= 60 else { + throw ValidationError("FPS must be between 1 and 60") + } } guard quality >= 1 && quality <= 100 else { throw ValidationError("Quality must be between 1 and 100") @@ -111,23 +123,176 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { defer { signalObserver.invalidate() } do { - try await recordVideo( + try await recordVideoViaStream( simulator: targetSimulator, outputURL: outputURL, - fps: fps, + fps: fps ?? 30, quality: quality, scale: scale, cancellationFlag: cancellationFlag ) recordingFinished.cancel() return ExecutionResult(path: outputURL.path) + } catch let unavailable as StreamUnavailableError { + FileHandle.standardError.write(Data("warning: H.264 stream unavailable (\(unavailable.underlying)); falling back to screenshot capture\n".utf8)) + do { + try await recordVideoViaScreenshots( + simulator: targetSimulator, + outputURL: outputURL, + fps: fps ?? 10, + quality: quality, + scale: scale, + cancellationFlag: cancellationFlag + ) + recordingFinished.cancel() + return ExecutionResult(path: outputURL.path) + } catch { + recordingFinished.cancel() + throw CLIError(errorDescription: "Failed to record video: \(error.localizedDescription)") + } } catch { recordingFinished.cancel() throw CLIError(errorDescription: "Failed to record video: \(error.localizedDescription)") } } - private func recordVideo( + // MARK: - H.264 stream recording + + private func recordVideoViaStream( + simulator: FBSimulator, + outputURL: URL, + fps: Int, + quality: Int, + scale: Double, + cancellationFlag: CancellationFlag + ) async throws { + let initialFrameData = try await VideoFrameUtilities.captureScreenshotData(from: simulator) + guard let initialImage = VideoFrameUtilities.makeCGImage(from: initialFrameData) else { + throw CLIError(errorDescription: "Failed to decode simulator screenshot") + } + let dimensions = VideoFrameUtilities.computeDimensions(for: initialImage, scale: scale) + let bitrate = H264StreamRecorder.estimateBitrate( + width: dimensions.width, + height: dimensions.height, + fps: fps, + quality: quality + ) + + // Constant-rate PTS layout — the eager stream is fixed-rate, so lay + // frames out at exactly 1/fps to keep playback smooth regardless of + // when the encoded bytes happen to arrive. + let recorder = try H264PassthroughRecorder(outputURL: outputURL, frameRate: fps) + var recorderFinalized = false + defer { if !recorderFinalized { recorder.invalidate() } } + + let streamError = FirstErrorBox() + let pipeline = H264MuxingPipeline(recorder: recorder, onFatalError: { error in + streamError.set(error) + cancellationFlag.cancel() + }) + let consumer = PipelineDataConsumer(pipeline: pipeline) + + let config = FBVideoStreamConfiguration( + encoding: .H264, + framesPerSecond: NSNumber(value: fps), + compressionQuality: NSNumber(value: Double(quality) / 100.0), + scaleFactor: scale < 1.0 ? NSNumber(value: scale) : nil, + avgBitrate: NSNumber(value: bitrate), + keyFrameRate: NSNumber(value: 2) + ) + + let videoStream: FBVideoStream + do { + videoStream = try await FutureBridge.value(simulator.createStream(with: config)) + } catch { + throw StreamUnavailableError(underlying: error.localizedDescription) + } + + let startFuture = videoStream.startStreaming(consumer) + startFuture.onQueue(BridgeQueues.videoStreamQueue, notifyOfCompletion: { future in + if let error = future.error { + streamError.set(error) + } + }) + videoStream.completed.onQueue(BridgeQueues.videoStreamQueue, notifyOfCompletion: { future in + if let error = future.error { + streamError.set(error) + } + }) + + // Give the stream a moment to fail fast on an attach error before we + // commit to it — a failure here means H.264 streaming is unavailable. + try await Task.sleep(nanoseconds: 1_000_000_000) + if let error = streamError.first, !pipeline.firstFrameReceived { + throw StreamUnavailableError(underlying: (error as NSError).localizedDescription) + } + + try await runStreamPollLoop(pipeline: pipeline, streamError: streamError, cancellationFlag: cancellationFlag) + + // Finalize the MP4 before stopping the stream: the moov atom must be + // written before a supervisor's post-signal SIGKILL can land. + pipeline.finishIngest() + do { + try await recorder.finish(stopHostTime: ProcessInfo.processInfo.systemUptime) + recorderFinalized = true + } catch { + if let streamErr = streamError.first { throw streamErr } + throw error + } + + await stopStreamBestEffort(videoStream) + + if let error = streamError.first { + throw error + } + } + + private func runStreamPollLoop( + pipeline: H264MuxingPipeline, + streamError: FirstErrorBox, + cancellationFlag: CancellationFlag + ) async throws { + let startTime = Date() + var lastProgress = startTime + var warnedNoFrames = false + + while true { + if Task.isCancelled || cancellationFlag.isCancelled() { break } + if streamError.first != nil { break } + + let now = Date() + if !warnedNoFrames, !pipeline.firstFrameReceived, now.timeIntervalSince(startTime) > 3 { + warnedNoFrames = true + FileHandle.standardError.write(Data("warning: no video frames received yet\n".utf8)) + } + if now.timeIntervalSince(lastProgress) >= 2 { + lastProgress = now + FileHandle.standardError.write(Data(String(format: "Captured %lld frames\n", pipeline.framesWritten).utf8)) + } + try? await cancellableSleep(seconds: 0.1, flag: cancellationFlag) + } + } + + /// Stop the stream, waiting at most ~1 s. The MP4 is already finalized, + /// so this only politely releases the simulator's encoder. + private func stopStreamBestEffort(_ videoStream: FBVideoStream) async { + let once = OnceFlag() + await withCheckedContinuation { (continuation: CheckedContinuation) in + let resume = { if once.trySet() { continuation.resume() } } + BridgeQueues.videoStreamQueue.async { + videoStream.stopStreaming().onQueue(BridgeQueues.videoStreamQueue, notifyOfCompletion: { _ in resume() }) + } + BridgeQueues.videoStreamQueue.asyncAfter(deadline: .now() + 1.0) { resume() } + } + } + + // MARK: - Screenshot fallback + + /// Last-resort recorder used only when the H.264 stream API is + /// unavailable (e.g. after an Xcode update breaks the private + /// CoreSimulator surface). Polls screenshots and re-encodes through + /// `H264StreamRecorder`; caps near ~8-10 fps. + private func recordVideoViaScreenshots( simulator: FBSimulator, outputURL: URL, fps: Int, @@ -150,54 +315,27 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { ) defer { recorder.invalidate() } - let frameInterval = 1.0 / Double(fps) - var frameCount: Int64 = 1 - var lastLogFrame: Int64 = 0 - let startTime = Date() var lastPresentationTime = CMTime.zero - + let frameInterval = 1.0 / Double(fps) try recorder.append(image: initialImage, presentationTime: .zero) let writerStartTime = Date() while true { - if Task.isCancelled { - break - } - if cancellationFlag.isCancelled() { - break - } - + if Task.isCancelled || cancellationFlag.isCancelled() { break } let frameStart = Date() do { let frameData = try await VideoFrameUtilities.captureScreenshotData(from: simulator) - // A decode failure must still fall through to the - // frame-pacing sleep below — `continue` here would - // hot-spin the loop for as long as decoding keeps - // failing. if let cgImage = VideoFrameUtilities.makeCGImage(from: frameData) { let now = Date() var presentationTime = CMTime(seconds: now.timeIntervalSince(writerStartTime), preferredTimescale: 600) if presentationTime <= lastPresentationTime { presentationTime = CMTimeAdd(lastPresentationTime, CMTime(value: 1, timescale: 600)) } - try recorder.append(image: cgImage, presentationTime: presentationTime) lastPresentationTime = presentationTime - frameCount += 1 - - if frameCount - lastLogFrame >= Int64(fps) { - lastLogFrame = frameCount - let elapsed = Date().timeIntervalSince(startTime) - let actualFPS = Double(frameCount) / max(elapsed, 0.0001) - FileHandle.standardError.write(Data(String(format: "Captured %lld frames (%.1f FPS actual)\n", frameCount, actualFPS).utf8)) - } - } else { - FileHandle.standardError.write(Data("Unable to decode screenshot frame\n".utf8)) } } catch let error as VideoWriterStallError { - // A stalled writer does not recover; abort the recording - // instead of re-logging the stall once per timeout forever. throw error } catch { FileHandle.standardError.write(Data("Error capturing frame: \(error.localizedDescription)\n".utf8)) @@ -262,4 +400,23 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { return baseURL } -} \ No newline at end of file +} + +/// Bridges `FBSimulatorVideoStream`'s H.264 byte callbacks into the shared +/// muxing pipeline. Not a `FBDataConsumerSync`, so the passed data is +/// heap-backed and safe to hand to the pipeline synchronously. +private final class PipelineDataConsumer: NSObject, FBDataConsumer { + private let pipeline: H264MuxingPipeline + + init(pipeline: H264MuxingPipeline) { + self.pipeline = pipeline + } + + func consumeData(_ data: Data) { + pipeline.ingest(data) + } + + func consumeEndOfFile() { + pipeline.finishIngest() + } +} diff --git a/Tests/AndroidRecordVideoArgumentTests.swift b/Tests/AndroidRecordVideoArgumentTests.swift new file mode 100644 index 0000000..220a5ed --- /dev/null +++ b/Tests/AndroidRecordVideoArgumentTests.swift @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +@testable import SimUse + +@Suite("Android record-video argument construction") +struct AndroidRecordVideoArgumentTests { + @Test("parseWMSize reads Physical size") + func physicalSize() { + let parsed = RecordVideo.parseWMSize("Physical size: 1080x2400\n") + #expect(parsed?.width == 1080) + #expect(parsed?.height == 2400) + } + + @Test("parseWMSize prefers an Override size over Physical size") + func overrideSizePreferred() { + let output = "Physical size: 1080x2400\nOverride size: 720x1600\n" + let parsed = RecordVideo.parseWMSize(output) + #expect(parsed?.width == 720) + #expect(parsed?.height == 1600) + } + + @Test("parseWMSize returns nil for unparseable output") + func unparseable() { + #expect(RecordVideo.parseWMSize("cannot connect to display\n") == nil) + } + + @Test("screenrecordArguments omits --time-limit below API 34") + func api33NoTimeLimit() { + let args = RecordVideo.screenrecordArguments(serial: "emu-1", sdk: 33, bitrate: nil, size: nil) + #expect(args == ["-s", "emu-1", "exec-out", "screenrecord", "--output-format=h264", "-"]) + } + + @Test("screenrecordArguments adds --time-limit 0 on API 34+") + func api34Unlimited() { + let args = RecordVideo.screenrecordArguments(serial: "emu-1", sdk: 34, bitrate: nil, size: nil) + #expect(args.contains("--time-limit")) + #expect(args.contains("0")) + #expect(args.last == "-") + } + + @Test("screenrecordArguments includes --bit-rate and --size when provided") + func bitrateAndSize() { + let args = RecordVideo.screenrecordArguments(serial: "emu-1", sdk: 34, bitrate: 4_000_000, size: (width: 540, height: 1200)) + #expect(args == [ + "-s", "emu-1", "exec-out", "screenrecord", "--output-format=h264", + "--time-limit", "0", + "--bit-rate", "4000000", + "--size", "540x1200", + "-", + ]) + } +} diff --git a/Tests/AnnexBStreamParserTests.swift b/Tests/AnnexBStreamParserTests.swift new file mode 100644 index 0000000..9976fe0 --- /dev/null +++ b/Tests/AnnexBStreamParserTests.swift @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import Foundation +@testable import iOSSimBackend + +@Suite("AnnexBStreamParser NAL splitting and access-unit assembly") +struct AnnexBStreamParserTests { + // Representative NAL units (header byte + short body). Bodies avoid any + // `00 00 01`/`00 00 03` run so no accidental start code or emulation + // sequence appears unless a test adds one deliberately. + private static let sps: [UInt8] = [0x67, 0x42, 0x80, 0x1E, 0xAB] + private static let pps: [UInt8] = [0x68, 0xCE, 0x3C, 0x80] + private static let sei: [UInt8] = [0x06, 0x05, 0x01, 0xFF, 0x80] + private static let idrMB0: [UInt8] = [0x65, 0x88, 0x84, 0x21] // 0x88 → first_mb 0 + private static let sliceMB0: [UInt8] = [0x41, 0x9A, 0x12, 0x34] // 0x9A → first_mb 0 + private static let sliceMB1: [UInt8] = [0x41, 0x40, 0x0A, 0x0B] // 0x40 → first_mb 1 + private static let aud: [UInt8] = [0x09, 0x30] + + private func annexB(_ nalus: [[UInt8]], fourByte: Bool = false) -> Data { + let startCode: [UInt8] = fourByte ? [0, 0, 0, 1] : [0, 0, 1] + var out: [UInt8] = [] + for nalu in nalus { + out.append(contentsOf: startCode) + out.append(contentsOf: nalu) + } + return Data(out) + } + + private func consumeAll(_ parser: AnnexBStreamParser, _ data: Data, chunkSize: Int) -> [H264AccessUnit] { + var result: [H264AccessUnit] = [] + var offset = 0 + while offset < data.count { + let end = min(offset + chunkSize, data.count) + result.append(contentsOf: parser.consume(data.subdata(in: offset..p^sYg(wylBM(|A@IR!3;2TNdQTTOb6c9jW+(V&ctjX{;1thEkBx{- zrdiYJHs;oJ>m{%-Dmj^BXK9(5nriV!ba+I3Y)G6%eBw6CxmGNql6SV}3`t53<|M^LbKtB0 zsUU$993L5(6p?IiZcR&$N`zk|L1|*+<6}ai;7#zqWmdGL*yu1Q%fAJ5T3q6A_*~J2)Y~B!VhRP7H~TgH^y66GP_SiA)UH5s?J1 zQX`_bMJ03KRZK)Wd~CnuKNrC}qT^uINnsIj5n(CG_O|r7W)ef@RFW8x6a~LX3=94* zap8A~VfJB(k+dD5P%U#(!5e#P8w)xuVNMo%y2Vm>$(gJ6*EuB3e(7>pkdz$3vA3l~ zbD)l)J)sF;Qy~e^d~?PC(D6_aMR`i!LV2L=%P|R8RH`Nq2)9|lOo;-pL?K|!Gmv)y zXszpJHRbK?}!iO2|p07YAVJO`jq z<$$7r621Yh{aL4k7=zaeYAuJ=2P9c1Ujy}u-3f9>f*k?o{(z&3Ho>pc?w!)d8x>S6 z`@;>cQkxeGHVw)oW=R7FW_=HT#{&rq1d!4&IRGO9v~Md>v04>$m93w_2T{mGSm_bf zOI+6Vm-Hc8o+pTU&Q+2s=QywcQUq|VwE*xbfQV^#SS7*;}XGME- zrm<8wsQ7M!dn@iL^#uU`Z(R*#;@rs|bmwz6_mUHt0w|ag6R0$KC{Q^H#DQ>shtR6; zKue>?@1(SE^3I-fd0^$A6Z-nXl+q*t_?Fs}SBQGC=uty^xnUT_r1nG$(&|@^3cI%S zsK3%fDu)LI`uSLZp0ger(>XT$_UyBhpzWP;B*~XT%`IcKb)iv6y*S*r&jwp{N8ceTDk)xsGZzwU`v;I$2iEK435Mu5(91!y9yNv(Ah{OEOc?JNf!N}N14%0fYi zE4Zf~}YX&SWO?z3rwEjqbbZeDIWj4@5xXqlrR{ z?)9JRzr4O`n(pw&EFe5l?4uNYOjKpNnhx<}_A`>=U*HQZ#hW=`JX5!JrP<@z`pn7e z2p8er1!&&0_}Y$R)eEd~355uy=M9_6Og zh8{}x%=_NCvr1eR)QP8BVb1)@HP=q6rcSjw`G0Afgz zs(100#o3<9o%`}mn1_`cEMxINkyZOCxskdV|1U2+RavO(p0AnCEsxL&%k=*AS=RCu zrIui_ePgGl5M7Y>$I5+F$8ut!0&H{fe*W@9_g{}+UTZA>Zw~Doa9Iv&`7_E64iuv#0@CbOQ7IbjYrDMoofr5hu0ZaMe zAOt+XSzgY&OaTg*3*@59ZM9~%$tawb83U^p-h#~F4iB2sc*g2Y$u3ocLfY{K({6~i z*UI6Taq5ZuDMU&af&1mXYJWa_Z(2a|WB?U_zJV7UZSHA{A{1kLP_L=i_Nn9>&J2+3 zMA$66dU<>QDBnA`UPHSlw=9CQYA*QPpv> zOW{pmnZkK-f4Y%Q%NW)Oe4p0Y7peR8jy>HY&_0RomBBvs%S9%jiy+7}j$Opbsel5r zF#1&#vpj$n4J#=xC@k75C4-#ZmD(uWd`p?0OkTi7X`tb3Oae>cL)ou#BxdLB5P-{I zxA`n@W{v?bVlj7_sfA9B80EL%?#XuBM%}uMzDy9U(n;>c7Q>=*5o zdoen4Xu5n5d|3B>D@FAj&zf!Vl5aibp;2SE3AGPJ?(pNe-;vU&gf^#jb?08-;k1zo zFa>}Lok#^nO3TZCpR4zN(Lr02`C1Y@N|x(5{i#S-O-twyphtd6pzECQ`)d8vRHV)9 ziP~(cK2++JCv3w49Jp|^eV^qvk%)q+z9pg(rWF(X_%lVi*dhwm9UaT6&QuT0`TJLm zfgNf5QEU;-nyngIl1iy2l>RmFyE`2_ok0PM4Jbw&K%uFbd76tva3bL;L&<0$dhZyI ztv75N5SWmH>7m!`SI!G#2qyUA6|1lN4=q)iR!mI;0U__=g11^DJxwNT4;8p5 zS3rE^CD8^#kbc7794VYuBz~NUoozuxt-gLVD%D-ic1q0t7FgmW)LniA#sGUEPT8TVeuaNNwXRfMg>N^dzra1jl?muAZYG)-}6 zc%K0C5wQZ=^|7mmm7f&5IL1`5S75<@fW<5jixG!2WLKx|U~@eR1w*3)B3TG1msH5b zIwAqkg2Of=fd#Aq8kLNUBh!iss8=^)27Nnk2xoChcA>8(r>!{m&}qv?rXMZ-5l{+2>^RRZ;C-z0r z!3q>y5GYzSKy*R%P*nVqvDnHqplYon)E=G(Xc*x=9uXx9+bzUs-uHjSU8Jm>z5J69 zt>d4u<|?3k?*C_py7W{FX6mSLk3?^zeTisRZBZJN+YGZB9{L8%C1B2Bf(I|_0ZVQJ z+R9v*6aQYLveIhnEx}xsTll<&6Gyd`l1s|RnC!(G(J^Q2PnzNm6R#VbuP<%%=);vzoP*)fXMh;oVLXC3#<)?E8e&{r`oD^ zD#EoeP|BPj@>jRG5o}cZv^QG`%OCGSc-QKNm1M65WlW?+cp%;DbJ(#=De$lU^l^{g z%`6&7z`$h-<%Z$a)C&j8>e1>KOKJ=A{I}}5kiw-g2gM}^7D?NC*<}R;uW%1cue)-O zsN1Z#4D~*{HJTMOb2^!v*~F;z!P_ zv+hrBz*@nHathEOtQ-Y?>!w9Ex&6l&K%C$Iy8I;8>-1Q~a>~1R>xJlv#%WwxVz!W! zf#@NUla|8S525L=^HQIZz~$^#wt2@O!@!FmONg2xEZOZ)#>%GzI^ z{72o%knDK}WE?hXi`6+wb;pDu%1<*Sn!k5tSWDDj-_TJ)!+g4^-9b(SdRV>BYI&#? zVgS=`heHfK(Up3ZGH02dyd#x($>a^F9Q(K;xMl$|G5cdcuC>ZCXG~Od@YJIBpMzUF zTa`4`9xW7gR|)ywom@IXZ{x;>26A^b64Za(w&@6MDv_B>)i&L~=t1?RRI77Ey{#uT zFUv@$_7d0ZvT|tOgAZs{ShYH4mz5h&bKFK|N1f7Sye^ujolJP&Lg7p z_S-=8RC|6V-}%wIKP_h?v`61eYao+br>88o&-!cpGdqqhIHYcyaf+AuDZai@nb2G* zS=%c}oHUnM=J+LsMv>ju>*}`m=(a!k=;mkhtH|bj)y+Tc#AgG)4}Nz&-+pt!0A3lZ z_)m-CKh0?i<|*woQ=0aF@=wJ-EfxywJeT!sv!Dyu@I}jcDeBxo94Ue~Yt)k-KT@Qt z9kJK?E%|IWJ1rH!w1shbs_hPn|0BC)VRTm^kr;L(dNNwRTweX?rKWEne52?EgUz8McjHw>iB> z?^OT&TCzCDlwVyNs6u}KH2!DLqT-(~jDH(%6G5W*lJusT#4O-%UQ{x^wIm~SVBtNy zN5Q&X)EeF}c~`KCyL9aj4`IR-T$V4YGfTMmV1oKjN!(P9aMj{=xnmNG5pkW*bLW#C zetD0jTB)M`DshDPu^Ls-MZ@}xqqm78x<&6K2F4~6?^+A96>m<-7k6~53<%f^lb6#) z?nSCAk5Q@3TFU3XDHXZPutfoJLyf7NgO>Hd-c?e6o9m3q0X2_QjMCAvRULOmQuYrW zYg=-`|A14P^$XIV7b@(%L*;}+rL$JNot?^XRf1uyTQ+HM%bQVI(N*&7IJ`|~)<@n} z_9%R$H#)z2ag;1L=iv73iAMgERCVs&BIj<2+}M-+HZ9Pw;q-ka+;qC2X~<$oP5^~0fQ z%VQq5whiPmeB+y{Y)AUlxp>Z5=O)vR&^yXeht{{>G)hI9qDE&{Z9RHNwEiO<<*ef| zL2+EDZl%*7ACJ$X6yx?)t|c{=yC^d8^yj_~Ga9|Ul+=p|o_@jn+v5yFvxmdg8KEl% z1`k%NEMt^sx%u}vP*-(>wL4W-Am0VPs*?O--fxxi=2Y5S)mkO)`K3{C%&NlZv8AZi zw7_E*HFrXz#)dh8C&%B_>xrpr>Y*>EBy6U+i<(c62iI;~fwFQ=67n58bt9mok`8-u zFzv?NVbY(z`RmVpu8nAyt3Yp)k(x=TRdEe+e>TV+g)>&0aR>GIY6I+JiZK@-mDr)|=Oy$J{6J9DrYW_Km4S)+tM55K98xtfDmd)#d{&(j1<8?@!6f!;9i2HX(kDK-+popwSic zO5G1NV^I@9{DI7E13kXm9!jJeZm&B%FzCJeXYifi12aR0b+PMOa06|IuCHwOY-rX} z*Gr<4P4nRav4oc71 z1Sl5WZ+Dj%Ymt|< z>%FFjfmzGezjl5;^|=|5B8XTnjRpKLC1#Kqslv+G7~Tdz{oMb=SV;J^LtvI9+u?I@ z248z|0_YiN1!5x?eA);w?cOJ>1U0iJ8!u7y9G@9j0SeE-BDhWb%o9*lc!tfmLi?Y= zi+y+h%mswI03)LH%n?Hj59FJqfgc6q`*Knq56|nz>p7W4dRkxd6=|%`B^waK3T!=# z3cippD%+zzj15RYhV`Fa zO2mPe_yuRWLx*bsij8u~kA@iZ4^_78eP;|OJRT^l5pBeyXNp( z)%Q;^sLTe2%hOJSehrpK;b&nmb~9I^jBNlQw+1*n(hkCmScwH#xv=*DXu#M~AKJ#& zS@nr9d;}t$Z(nF~3~=Sz`6Zlxa`r$>yXE!sLrsaTZX*ILX8RhE-bg)t5C%^2Ds%lc zgl++|2!($w%0^B5hvUk)e{tP^=%o@t+OACnwrb3i^2tEMpwj4K4Tdt1ft# zbN=*MNNSJX6pIoNcKs5WTz_D8Mfi2qEsRR1%#9s4(HA#Uv&%G5m+)8xLRVgr2)~``9T2k%Zj*!i9V` zN7ZNYl?*v{Z~m``3znqz)*)$aDK8OZ5a6O%P+;4&nD?9g(eG9vx)`6Nt+q#+f89{? z0(4JN6oe&8AIG(b7iyQOyuav|m#nZ+9w%}s9Ji;h3rvyeeQ_cGj{$X+QfyjrWzQQ4 z*s3&9VW=+6CCa(5Ss)9K)#Bk{X0YZ?>C+xlN$n2^Z@XMfYx3UTS?mj(%x!kdono*l zEtCAhZs}4_Ag=7Z@K{M1xsIKb^)usz$zg4;=)b7Ucu8FP2q%kr(&*Y{0ysbc4W z7fE#CA7%1*?NSP)FM-LEy?p^2JG}}>OYX61E?J|G$8@;%NxhlF8id<&N{L-Brc~=% z2ISUbflHVZX0k2r?oj0_swQ?b$EhbOF?_Y<-DA$GKjz?VWrMP}N|LPq?&2d~-H=(K z$Hv6h`?&QLhqkQjn}IBWUvLwSVHF(oW>;*#>HMi!~A-c=cIAB=W z@muDtlMAc4aL*GBdem>DHbu$pD__up?t^)P9PfG-*hE^i zsntFOg0jQ%=qRuxG~dYXO7KS(|6|C?qKCJp5Ek`#-oZ-=A@NaQ<^1`L?2DnEwTQ%u zR0}{(`M@L=w*Fh~f%e=^B>W_%W6>95IUcI3-2C;9lq)+tklZ1m-(s&`&E-E4JQb`G zX(oynj%{gU&WfMaq%K~x+goL?nrhz+CF`~-dbct@?A7`at87GqsaE`R^`?+?6IK?a zjRJ~>gO$Alq{h8{zdMeI+i-#sOb%Y7iviRC3ee@lY3Oho&_%rc*(?bWq71B7=&!!Cux6FkSNAsF zT&jsnRCHLsv*t(PUG^ccyvFfGfWLNo&4M-_e(4zu&VU$yJm&P7g>YPFKn@FW>^KFP z!LklA;1ETD@rSv*xxGEIQp7N1xyXbbUkJGgkx*!j^h(mQ!Sdy#sf5YZU(I;k!%MJp z=KSvtz8f!m>8{G={DKRem!UX3KvB}}hFgd7oZqGG@F^XpmLhV*q|F)gy26RVR$$CS z&)aYQkBij>$IY8;C$Ugt|EZRV6p2U_bSxR*S%Afu1K|G=Mnn}ALeaXQXoSm%>@ATE z%46gH&bijJpH_db`d^ZZqcDIv;9Y$PvW_~ry;vLrP%>0%wp`;=_8x>#YXZK{U~a~` z4cz(PErkd+6AOvjMI5jgpn-ctJE$fCf-sq&FK4!#xF(wbu;lrWQ`0!FxMD=?g8?W? zJ@j;8N?Unel~@Gr7lC|@Z!D#Qs#ADHzl~DlE!b@YSVO?qZCHtUn zCPOAm77e-a@9K%GQ0M^^<6Ta_JlD)Rc#Z@+9G`sd zXsFF#(>*g-A5haSr41#auwhuw?!GgjKuCiSlR$O$%y3caxco*N4b&c)Q&gMI;j9X< z)_d;qmL3=wmq=pz+L`AhaJv008MOK1P-@;>)t7>Ti_+7maE0GAw}J`-&`?=4K+%Nz zAUFV6v46cZo4?xs>2F}GM_K7Kj%i;>Dy5i2VDUP!=}a1B+hgQyc*P}YbL{t*a0-N3{LKr%oHV@D{|_%BGRbb)IbV}W4^EDrzx{(1R zBYu&-ZJ5ePugc5bp(DFg-y$lGTaBvD7OzlOd$20k<6_Xb(JsBAH9@-*$UA&|zB68j zJlUYMUERm`!Ph|^SKK4e28EO(m%W)oidu1afPn)WCu9S-6_3s3@sqW&xVI!?{C=7x z0*Ub1)gKE=8{UX0{DN&g$y3?6s)!6idw&rl&vIQbaW!DU;Ed*}DCf|n@wrI=Xar=$ z%COT$)#-RPa5??dDzU#%g2-k&E^?UXl_6zo%IKRQAhD9pnMRqAWuOaD6WllBelXR66Pckd_cRn80EZ#jM zm5U7gy0s5^OoAI5X($>8P&U9AQ*SiNvJ#{6L@U-0m0ZMAGJ86hk6PJQ8oN=A{PA0f z7;J;#XowK7R08rKK6E7*8g}pOd}To+0s{<;eBA+x9MY3PF1*bKIxxy(P(>gMU~yA2 z__ovg>N9N?6y92ji@*|>Hm_*j&mn9V&wUP-J1yi5s^xcQSwK%1OLE*VfBEO@TBs9hY|sbc9rNES2F?8+cPmw1NCgMC&q52StrZ>9t^7>tHv0SUcGdAHN?@i;31 z@B>(I0*0UtHS*~{N7|L1Isbh~v1-l_cAa?m%y=6ssbXT?&9}x%GG`Ye(s?B1e0-`D zXkA^ids#pS{iF7MVX5u&x$**r0{1=I(w8Kz9JS^nbYFbQ%Cgq@dik)QHDJNLYk z)hzIB;m%&5!|iN{($72oY68aLzyNE>B7%ket%q`|T8YSht4-@XoT_NBmFpGm+jMQ6 zW0g>2T1}wiSUt|+qufFh11B<#3;B3iMk=aR`E%vhyPgoj0}cbs??TBqaIzpjQnQJ z6Q3UO2fjvi^7L=9Zhh|Se?tLOv>XhoDA1pcvay|@q^N~|2(OwdmD>#rYmSLYtMNLi zXY@~Bx}9ghoKbRFb(4MYPCb+~%|eNdeQoxzs4q(6;{&74mXC8m^J4}as_5NGJPnj) zeAFTb+oX>`m$-A_EZ%n2HD*?ESD~br1FpNY^k}GGnB@^uxLc4hlQlk5En||i6(6p8 z%zQ%gaC-+N2b9H)nKA%KP6dD_!aDxS>E?JdnytVi-h&DIedYCMhZBAN2>V#SWciFp zh(1e=%UQg4t=~hmSaa~J-~rio?DhP*E@cYu{Vob8l$Hm;LW~MM5+-Uw@E_njc(fJB zrKWQA#K$;>FR(AT_r9^$&mf&`EAP@4W>mI__gM0@CVgAJp1uz#!P9lJF^LwV4uocp zbh#luc9IRG8(ErMtaHa0M3w4o1_^$bz>1-P*8E`0+&zPrOw|Rds!fdJKW;ee`4tq+{BR>oAp{;x|7#5c8?;n7=p->zlpMTSm&Br=gy;cBV&8QpNo& zw*m9^B!A5NEo9HSANFjxpe=T^+UR&{4=|$@VwFatnK%eeY!3_?kP(K53bNtY>GEs`-PisqFg^B%3V;3{)w;8Dvk&Wg%oJ-CoY<|*Z zMmsfzCh}$%LV$Oh+G@t=rf(~ZG@XZ_Qif++OBY839IWcPeaKuU7q;6Q^6kaT*$H0V z>Cf|&RWA!8qpF;9TRsyxIvXoz2TPsR?{a>dbPhwm(Mpt1>;?s1O&2lC#^;#c&8BU; zI-SMeoE?C_qPnaqR1M;&1b_8WNLSPyXE`H@u6A8e#H?# zt9<7bA3KMZ1ih@{=7*@X7Bd+Id!C##K3%M%GG;028$71g2=gk8jTyYW7o zw1{(zNRP~mb!39TXP?HYMdrnlGr!f%UI8KMtvj3z@{Osf8MweSdj7gwqwgKfA^dMG z3@%8!u-Q5J<@t`QGWll3^RK0*&r`gt?G{z|A~?8JU?(bp+b=qzZc%?-tDKdqsdIPE zD1x2!@$uy^7PZ0QpiDZJN=Z?+F&OwJBd;p8jBUm$4JNF(b*m}=VeQ}Ck=|{`pU2hx zfxDEd#oE}yy0Rzntj9$z>mAgn>+m%F?W=t~ zT92r-rH7}-FE?1)-8Y#d(a*0elsOpm(8pR!mDl7w=G`0+2a}#>RV-cb=FqW}k$2HK ze%hvght#eZvTLwFq>eZUG)s8DRl!>ZRncANdB*JTspOGf4J2RT*67@Cf^C{>G5)t2 z)iW43fP1!f)-`|SyhI6Orl9G#TDS|BQvCzHQPfr?PWb*5?pw^FO3uw~gLoTt+-Y!; z&dZvYB3UpxbE7x7AoIpE=ZB{eG*%_XtART`pZgDzl_2+JuRbYjHJ8WcAyy$N!w#X(xK5erpsW-{}EM(O>D0KK8HrDrz|*BovKJOyNyzJ;N@W zd}%fcT5l|+G82{W<@8;{ue4bbsJHsITShi1r%$|so((d{{p2k?Ogx> literal 0 HcmV?d00001 diff --git a/Tests/Fixtures/sample-annexb.h264 b/Tests/Fixtures/sample-annexb.h264 new file mode 100644 index 0000000000000000000000000000000000000000..af9b4635f93fec6c8d128032eeb7b05a5e014632 GIT binary patch literal 7231 zcma)=c~}!kxA3czPDmg?ScI@igNV2QVMk>Ni-Ldyj_9DGB!sYxKq4RnMQCKhlte{kMj)W0K?40XQI}&CAKm>R~BDRP24685tEix;i7a z2$!G2LmXUPkQ66JM@J;u$;FlF!oe#7?eGTnE5nur*g7LVYyI$+F+46_;l~$bCh}s_ z5Jx7{$<~qS=z%j=;cG}9!RVHyEleq&Rz2rL-O7M z8y1LU1ZojWsAZu=>|y*tNKvCr_W5cr+Ca(v}E zjg#kI@l3XXfGiEr*$T+BgVyZW`*xb<&H@2I4d~MW#Tbx)X9K*K;IlT9dCEymx%D?} z`_1Wr8x3M&@tE^w)-2ia(60whFkN<%G6!$m(V@~img^m;#V|!gU@bs2+JO!t1_%u_ zNMybFk_Lu50hIZO=`h=0Zzq6H6gh{Lg#pvodjOHsCPA_Nqhxs*3;$ z5rEuJvo*_RYnIIPX3Ho9hskc0mP0R;d+LzedhGpB{i1NzxJz@-0S#Y+>*&-cWzJq~ z^c@0xa_8j%2oeJsDIX;m#V3S)@;$h)#m;gvLkTwT6D5U(JH#5Xoo_UlfhLnlF)#gS zeXCgf;BD}?oqzxxF}ysr7OSr*s~hRWFzxlzC^jrYQAps9053q$Xh#WtYIHq7l>2X3 z;j3ra4vX0`2+S%YqK)o4$VIM{<2yzaS*NN*3LMytFG+(Q>qo$qUcsw1VEesxaAajY<&1$(mlt=@R!fbl|xmt`oGi+1-?~z zcVgdE^@lMbVQ13%Z*DJDz?QOT;O^Tu1!r16YH07;*l?k$%Ns-AusC$see3I9@;@eU zplZ7vSxme5_)roD8H+9xS9yDjy8xZQteiXjY%)k67J;jsrN8e8+=Yd~Jtv-p1Rbb< z*ZT5T7IUQ7ZBs{GU5Ud1F`(-L8fgpj>*aJeuG1{ujBH=NVA8jr;w`gf)e0z+)^BHa zXWs5U#agmqhm0#b?L-C*dOmV*+NmQnvyy9Bzr|=vZ z=x&_#WbKQE8;-&V6w!wK;dgiQy*|ay$lZl2GTG-$E-U_jY2H5asS&EfEe*CYt zN=~82F=OgYkV}!1W#zT}LT@EF&!Mlk7<73*46XzUr}Cb3yE^bKErJ7bE}ay* z-{pHoV_8r)`1ot!_!fCU_}=V2%mP3Kf?Qyp+pG4*T1=N~MzBqd7hgsZS?77C_8CD> zI!L2hIX}}V&X|0r(lD~@f8=BVaxdhz0S%Bgq;{F(rdQ7GMJyx(mq^uFmh}5$*uhDG3$bY0q5T>??*) zn?wbMZZTaFq@_q|DjlGJ!b~7*63Yl$hKt-Os~i z(Fi#w5-)#72c|&3z6){Hx1lCG=zeVyLqI1FvEB~75;<+zYqGNM@~&QirZxh?dX6qo zKbVu6qcmJD-e__=XDMIly}et^AO4FNC}bB5O8568ep{libnWk@wkH)U{S;#Nbf^=A z8|;s)Dg1u2;DLUGCOr!i=rF* z6SMVqAXTmER2kU6*6ybe0~$mi%Rz<8rfF$O#JQ3`kNtwl-*1Ah`_NbHOL_zazYakE zp^g|lSAllG9L@4#92&x=g{G|Bb>6tIU_)IbG9VBNaTyHLJ)qjR$s94PZn)^uqRG4M z)j^Cp7KEPQ6srK6?$f&_M)g6w1S0W7Y)F$r&L<=`c2JqF?=mOo8yn!?9IHXeUr`Ex z#xGehWJgB>2xNyY-k%u~_e1i@93|UC>xecvHu9{#(Ia)IQ*zu=%$b0qU4Q`OXA~eh zw-Kn8Rm!FSV}jahZ6b=C;N^jyHwncmHPDmJP0OVbt1FHRZSGg#{$S<=#Na33Y*X)f zgYzaSpF1MNR%Odgt>*IMsM9Yye!7t=^TzU}{W+hN-q+(Z z4|hosquxM3AplAPwXa$m<|k4eE&jg!E@hm~AYIF~Hu5@)mhG$?-dC!1&_8JZMiK6^ zzmXhmRG8$S=st;P-ihKd?Ww%81!}hSS%CdZHDKNc41YkQNgTJg?lZjN7c924J@s5f z=)7Aam4BrAhr1n}ZW9ebteC*zQS6r$*FBES-KIX>&!XJwR}f76 zE1$hzwh8XjXh`s!8-_<91jb4+hJBi~JQ`F7&QI-B_hk{5j-;4Ev%Xo~*6Gy5LnHwC z9C9~*lj&n^yxr&?syQ{aMy0G)0?&b0&vR6a>SzH3wNBUHjC>8UL%&xPGCSdNwnIpv zV>irNu6UyJKvr$QJHj%M3ZCc;c3wBwakJK};agbqi3qMj;xnp|m%oVN`oc|vEF9A> z&|UY>`R_YgY~h&0mLCo4AF`G!vq8-8)7FXlsnLurgrPep2c%{si^^!t)5{Vn3NNoP zkITS@Yk1>=2XM@l-mbyA!oSz$C~l-2SFSfGoQRWt@QL3*j8 zN-DqhO_^GLm4ar|x~^`zBFi&*ltIimdPrxjxqMH<6DOL_A9t!g+$XL3N2$8@&2)Osbt%2u z!J6_u#5$`>xq8vIz5&{b0x9Ge$4@l;)!p&bB}Mx?!j6g$13er);Sbf>-jc&oQ+IV( z!8q+(6-z+oNDkWPFW9JzHmQ;NU$VQ!rhh+(*-sT(ToYXVirQH?P2Q1MUnc*&phaI0 zxH9;zy+#g=N`7hi=j{ew*~1}j{aH==3eMLIch%+b(KgZYK*S3lW-(R|Jm_{W4eJ+99^fC=40ylX*3PWW>je8HN8kpj>HUx(~h zx}*%W3bz+CWnUT#`$xz(0u)yi?N=l9r@G2HyamCO^tL{F7HDcOUJaEU} z1f3h6&LgY_!~&qkVu=Xg4XBi~Lrk!dd-YeV!zv;QAz&%P9q0mL1AvVbbHf*zG{z1* z*l{vc+vLBa^%nO1GBUJ$dk;Y?3r1<%6@c?)#hM*Aa#SQOI+!H}2p)FBfIvk7LI7|% zu;;U_)hDN&Pcd%wK7O59pYeS7(@PFYB1R@@@_~Mj`}>_X$%bz`Pfyx!(V=v=IUVq= z6l2n94eAV?DN(SSjKc#}r#6)o*aCtaGI`er!AM)fT zFK><}0+-u~3+zi*A7bC~`|=laUS6HsH}v-9GtPY&u}Wq51_fvaX*lPp_oz~Fx4t=N z3f=U4YV|aRq9{QY#sXOYHlzYg+6->`+w4lP@r{WTQDz-8pFiG?fP4pkSI z%+m+7jsu8}E@snP>5=c_2K>EC!ff7tKo?sk-`{gr1CRUxIN8R|Qk6Xw-TogSGF6t=6^vf_xdxjyFL>wls`KfbKbnXnFD&f3DXusvffwCq|Lg8hM2had?@T%2_d)$8MT{Nx?d_?^r{~ApL1mXyx*+WpGdz6>@6xKf;acAE~VF&6r>hfjkoV| z|G`E?1R&;J^ve8_5bt0DDOw9Q+wF~UKC-F5`;LVB8Mn`k@xzn}uos}h9rr5dY@h_G z<9HULha-}_NFOt%UYAi|=@F9?1>ynqoYwy(Lao8Od}-hAJ9{vhv_K!p1yI=W+p@dA z(T!2*iDWfsR+E@zz)rphR`y|iuO(!lKYbF)+XuT1I0;_`5C1YfeV`6wngsCcfyGdxF=`^oTliWV zAgr@E1>xTSpZ7@L)pe!pQB~TS%&v33g}TbOZkCSibaH+;tqO!%@~bX%toEz!k;xh z1h-8*!c(yfAdY|l*NTo1sQA^1J0z}>Q2>ce%>68di3BqtbdO;CHAl?URO4g~sUc8a z)1e14hmdpUk>v^$1u{TUKLFLfBi}lie)O@8PY{HgrB2DxCZntO?A_u6fsROx^gK>Q zEY^DJXcjRV!T2j6rnvyX#V!)q`(Qo_(M=82amZYyI6*{-h^cEc1=9V5E8?+aZPTFG zoj%)0=o28o3A_pP5>ZXr+u;WHD#;PyFb~B&gcE7*BEq#&|3?O2lb@d7gmYgefRKL; z;Fw3ARQ8PAzpUNu*GWt=akxYF`?Xd@xPM0}?GjJ?Vw3_?^1H9a>|&eCsT;g5331MH zHN%|*nxHMR+g33Yx`-#I1Ny8mkPGVtPeKy?!?mVx>d0P^b^#y#d6T(<0^K)TsL-O% z4{26Vvc#C8c5=<*&N2KOW}Jol2Jp~tX@f30eWLnekK=mU;o|j}RO-Q`jx(1V@u{zB z^@E>;Hl`Gf>F7s|t+oczo5%k2(|jex$|kgO)w|>XLZN#z9*YEONq==QJ`#Cq=7#YR zz*)=0R9kD?@Rr5wDheR5O{o zW<#|_6T2U2L7=7U?)!k$0-bj(7Dxjy3AuTo;Fh)4Um+!na_wUEI~+mPMa@8 zHWNc`frCj1QH~mTg^RJsyb2H8XdD&j9NQWDw#4mQ7eg05VO^1~YW^Jf<-$V6EW(2c zu#lt2Vm|d0QHtoCyzQ4UfFo-2W-qlJZ3J~$;DKGV&wOF$ro|stgV_af*^nN)+4p&SZ*8UofLubdnmtZx5@GH`0B2gxl8zTFEPMwD z>KpGd@b6yAh(kUSVCo9~18njU$#QV#d!y>|n~4@6!G51&o;Hx}PYhWditd)j&zUc{ zy1};6_w*-wbsrzMmh*$%dc{Xgurd+|-tuJ`_Prc<@u4`F|hSTV$Y^U|{&&M6sIt~XgcBi=jB z0eQ&HD3yz5SE>2~as0a0_6#Anc*)tdqmt*Xcn*2dv@mfOIl{4I;{YC$GX^m!{dJp|Cd!J8i>98h+BORvf1Xy^Y8lUJYCZ_|Wc*TQud%i8m@xYB zy0Ln1<;LHpDpdAQ$1!=;gm90lRmmd+F6}kLvd-dMcJ|scMiF~YNbsJ`pYK-h@WRx6 zCt5=9R9=9*2S0zj`a>D^(iyhWCDi@&kup$NTXu0o<_u*EcP@+g-kPoOk<8t!ajnHg zD&ohiePQ6(0g1geny^BgDcg1K6-hE#e2J0m0<=W4#X(~6dPL8fmu*l)i+s`YLQlr; z40Jg0bGE_X8or8E8T}KF>e@`dW4a@a9+TqITa|?H0cl|TUH||9 literal 0 HcmV?d00001 diff --git a/Tests/H264MuxingPipelineTests.swift b/Tests/H264MuxingPipelineTests.swift new file mode 100644 index 0000000..2aaef07 --- /dev/null +++ b/Tests/H264MuxingPipelineTests.swift @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import Foundation +import AVFoundation +@testable import iOSSimBackend + +@Suite("H264MuxingPipeline chunked ingest") +struct H264MuxingPipelineTests { + private func fixtureData() throws -> Data { + let url = try #require(Bundle.module.url(forResource: "sample-annexb", withExtension: "h264", subdirectory: "Fixtures")) + return try Data(contentsOf: url) + } + + /// Deterministic 10 fps clock so PTS assertions don't depend on wall time. + private final class FakeClock: @unchecked Sendable { + private let lock = NSLock() + private var t = 500.0 + func tick() -> TimeInterval { + lock.lock(); defer { lock.unlock() } + let value = t + t += 0.1 + return value + } + } + + @Test("Chunked ingest muxes every frame into a playable MP4") + func chunkedIngestProducesMP4() async throws { + let data = try fixtureData() + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-pipe-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + let clock = FakeClock() + let pipeline = H264MuxingPipeline( + recorder: recorder, + clock: { clock.tick() }, + onFatalError: { Issue.record("unexpected fatal error: \($0)") } + ) + + var offset = 0 + let chunk = 512 + while offset < data.count { + let end = min(offset + chunk, data.count) + pipeline.ingest(data.subdata(in: offset.. second) + #expect(bumped == CMTimeAdd(second, CMTime(value: 1, timescale: 600))) + } + + // MARK: - Real mux round-trip + + private func loadFixtureAccessUnits( + _ resource: String = "sample-annexb" + ) throws -> (units: [H264AccessUnit], sps: Data, pps: Data) { + let url = try #require(Bundle.module.url(forResource: resource, withExtension: "h264", subdirectory: "Fixtures")) + let data = try Data(contentsOf: url) + let parser = AnnexBStreamParser() + var units = parser.consume(data) + units.append(contentsOf: parser.flush()) + let sps = try #require(parser.currentSPS) + let pps = try #require(parser.currentPPS) + return (units, sps, pps) + } + + @Test("Muxing the fixture stream produces a playable MP4 with the right frame count") + func muxRoundTrip() async throws { + let fixture = try loadFixtureAccessUnits() + #expect(fixture.units.count == 10) + + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-mux-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + var hostTime = 1000.0 + for unit in fixture.units { + try recorder.append(accessUnit: unit, sps: fixture.sps, pps: fixture.pps, hostTime: hostTime) + hostTime += 0.1 // ~10 fps + } + try await recorder.finish(stopHostTime: hostTime) + + #expect(recorder.framesAppended == 10) + + let asset = AVURLAsset(url: outputURL) + let tracks = try await asset.loadTracks(withMediaType: .video) + #expect(tracks.count == 1) + let track = try #require(tracks.first) + let dimensions = try await track.load(.naturalSize) + #expect(Int(dimensions.width) == 160) + #expect(Int(dimensions.height) == 120) + let duration = try await asset.load(.duration) + #expect(duration.seconds > 0.8) + } + + @Test("Constant-rate mode lays frames out at exactly 1/fps") + func constantRatePTS() async throws { + let fixture = try loadFixtureAccessUnits() + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-cfr-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let recorder = try H264PassthroughRecorder(outputURL: outputURL, frameRate: 30) + // Deliberately jittery host times — CFR mode must ignore them. + var hostTime = 0.0 + for (index, unit) in fixture.units.enumerated() { + try recorder.append(accessUnit: unit, sps: fixture.sps, pps: fixture.pps, hostTime: hostTime) + hostTime += (index % 2 == 0) ? 0.005 : 0.4 + } + try await recorder.finish(stopHostTime: hostTime) + + let asset = AVURLAsset(url: outputURL) + let track = try #require(try await asset.loadTracks(withMediaType: .video).first) + let nominal = try await track.load(.nominalFrameRate) + #expect(abs(nominal - 30) < 1.0, "expected ~30 fps constant rate, got \(nominal)") + let duration = try await asset.load(.duration) + // 10 frames at 30 fps → ~0.33 s regardless of the jittery host clock. + #expect(abs(duration.seconds - 10.0 / 30.0) < 0.05, "expected ~0.33 s, got \(duration.seconds)") + } + + @Test("finish with zero frames throws and leaves no file behind") + func zeroFramesGracefulError() async throws { + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-empty-\(UUID().uuidString).mp4") + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + await #expect(throws: H264PassthroughError.noFramesCaptured) { + try await recorder.finish(stopHostTime: 1.0) + } + #expect(!FileManager.default.fileExists(atPath: outputURL.path)) + } + + @Test("Re-appending identical parameter sets for a new segment does not error") + func segmentRestartReusesFormat() throws { + let fixture = try loadFixtureAccessUnits() + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-seg-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + var hostTime = 0.0 + for unit in fixture.units.prefix(3) { + try recorder.append(accessUnit: unit, sps: fixture.sps, pps: fixture.pps, hostTime: hostTime) + hostTime += 0.1 + } + // Simulate a new segment feeding the same SPS/PPS bytes again. + for unit in fixture.units.prefix(3) { + try recorder.append(accessUnit: unit, sps: fixture.sps, pps: fixture.pps, hostTime: hostTime) + hostTime += 0.1 + } + recorder.invalidate() + #expect(recorder.framesAppended == 6) + } + + @Test("A dimension change between segments is rejected") + func dimensionChangeRejected() throws { + let small = try loadFixtureAccessUnits("sample-annexb") // 160x120 + let large = try loadFixtureAccessUnits("sample-annexb-320x240") // 320x240 + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("simuse-dim-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let recorder = try H264PassthroughRecorder(outputURL: outputURL) + try recorder.append(accessUnit: small.units[0], sps: small.sps, pps: small.pps, hostTime: 0) + + #expect(throws: H264PassthroughError.dimensionsChanged(old: "160x120", new: "320x240")) { + try recorder.append(accessUnit: large.units[0], sps: large.sps, pps: large.pps, hostTime: 0.1) + } + recorder.invalidate() + } +} diff --git a/Tests/RecordVideoTests.swift b/Tests/RecordVideoTests.swift index 154baa7..da7c529 100644 --- a/Tests/RecordVideoTests.swift +++ b/Tests/RecordVideoTests.swift @@ -7,10 +7,9 @@ import AVFoundation @Suite("Record Video Command Tests", .serialized, .enabled(if: isE2EEnabled)) struct RecordVideoTests { // Regression for issue #35: under short-grace SIGTERM (process supervisor - // pattern) the mp4 must still finalise with a moov atom. The signal-to- - // finish path goes through a DispatchSource → Task → actor → loop pickup - // → recorder.finish() chain whose latency on master is ~70-180 ms; with a - // tight grace before SIGKILL the trailer is never written. + // pattern) the mp4 must still finalise with a moov atom. The stream path + // finishes the writer (moov) before stopping the stream, so the trailer is + // on disk before a tight SIGKILL can land. @Test("Record video survives short-grace SIGTERM with a valid mp4") func recordVideoShortGraceSIGTERM() async throws { let udid = try TestHelpers.requireSimulatorUDID() @@ -141,7 +140,7 @@ struct RecordVideoTests { process.arguments = [ "record-video", "--udid", udid, - "--fps", "40" + "--fps", "70" ] let errorPipe = Pipe() process.standardError = errorPipe @@ -153,7 +152,26 @@ struct RecordVideoTests { let errorOutput = String(data: errorPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" #expect(process.terminationStatus != 0) - #expect(errorOutput.contains("FPS must be between 1 and 30")) + #expect(errorOutput.contains("FPS must be between 1 and 60")) + } + + @Test("Explicit --fps records at that constant frame rate") + func recordVideoEagerFrameRate() async throws { + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent("sim-use-fps-\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: outputURL) } + + let result = try await invokeRecordVideo(fps: 20, duration: 4.0, outputPath: outputURL.path) + #expect(result.exitCode == 0) + + let asset = AVURLAsset(url: result.outputURL) + let tracks = try await asset.loadTracks(withMediaType: .video) + let track = try #require(tracks.first) + // Eager H.264 streaming lays frames out at the requested rate — far + // above the ~8-10 fps the old screenshot polling was capped at, and + // close to the requested 20 (a few percent under is normal). + let nominal = try await track.load(.nominalFrameRate) + #expect(nominal >= 16 && nominal <= 24, "expected ~20 fps, got \(nominal)") } // MARK: - Helpers @@ -167,7 +185,7 @@ struct RecordVideoTests { } private func invokeRecordVideo( - fps: Int = 10, + fps: Int? = nil, quality: Int = 80, scale: Double = 1.0, duration: TimeInterval = 2.0, @@ -182,14 +200,16 @@ struct RecordVideoTests { let process = Process() process.executableURL = URL(fileURLWithPath: simUsePath) - process.arguments = [ - "record-video", - "--udid", udid, - "--fps", "\(fps)", + var arguments = ["record-video", "--udid", udid] + if let fps { + arguments.append(contentsOf: ["--fps", "\(fps)"]) + } + arguments.append(contentsOf: [ "--quality", "\(quality)", "--scale", "\(scale)", - "--output", configuredOutputPath - ] + "--output", configuredOutputPath, + ]) + process.arguments = arguments let stdoutPipe = Pipe() let stderrPipe = Pipe() diff --git a/skills/sim-use/references/cheatsheet.md b/skills/sim-use/references/cheatsheet.md index 1bc1128..2dc9b45 100644 --- a/skills/sim-use/references/cheatsheet.md +++ b/skills/sim-use/references/cheatsheet.md @@ -122,7 +122,8 @@ sim-use touch -x 150 -y 250 --down --up --delay 1.0 # long press ```bash sim-use screenshot --output shot.png -sim-use record-video --output recording.mp4 --fps 15 # Ctrl+C to stop +sim-use record-video --output recording.mp4 # H.264, 30 fps default; Ctrl+C to stop +sim-use record-video --output smooth.mp4 --fps 60 # iOS: constant rate up to 60 fps (Android ignores --fps, native rate) sim-use ios stream-video --fps 10 --format mjpeg # iOS only ``` From 0b1f5c76532166acc4bb5d6f32ea61977dd47a69 Mon Sep 17 00:00:00 2001 From: "yuta.ooka" Date: Fri, 17 Jul 2026 16:34:48 +0900 Subject: [PATCH 2/2] refactor: guard video state with OSAllocatedUnfairLock, drop @unchecked Sendable Replace every NSLock + `@unchecked Sendable` in the record-video path with `OSAllocatedUnfairLock`-backed plain `Sendable` classes (CancellationFlag, OnceFlag, FirstErrorBox, H264StreamRecorder, H264PassthroughRecorder, H264MuxingPipeline, AdbStreamingProcess). Non-Sendable writer / subprocess state is confined to the lock, so no type needs an unchecked escape hatch and no withLockUnchecked is used. (Mutex would be cleaner still but requires macOS 15; the package targets macOS 14.) Also wait for the stream's first frame before committing to it: a cold FBSimulatorVideoStream that attaches but pushes nothing now falls back to screenshot capture instead of producing an empty recording. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Adb/AdbStreamingProcess.swift | 139 ++++++++------- .../Util/H264MuxingPipeline.swift | 96 +++++------ .../Util/H264PassthroughRecorder.swift | 162 ++++++++++-------- .../Util/VideoCommandSupport.swift | 139 ++++++++------- .../Verbs/IOSSimRecordVideoCommand.swift | 29 ++-- Tests/H264MuxingPipelineTests.swift | 14 +- Tests/RecordVideoTests.swift | 4 +- 7 files changed, 292 insertions(+), 291 deletions(-) diff --git a/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift b/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift index 2dcaa6d..59abc05 100644 --- a/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift +++ b/Sources/AndroidBackend/Adb/AdbStreamingProcess.swift @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation +import os /// A long-running `adb` child process whose stdout is delivered /// incrementally as raw binary chunks — the streaming counterpart to @@ -10,21 +11,24 @@ import Foundation /// Follows the same drain / termination-semaphore patterns as `Adb.run` /// (readabilityHandler to avoid the 64 KB pipe deadlock, exit-driven wakeup, /// ENOENT → `BridgeError.adbMissing`). -public final class AdbStreamingProcess: @unchecked Sendable { +public final class AdbStreamingProcess: Sendable { + /// The non-Sendable subprocess objects plus the byte tallies, confined + /// to the lock. + private struct State { + let process = Process() + let stdoutPipe = Pipe() + let stderrPipe = Pipe() + var stdoutByteCount: Int64 = 0 + var stderrBuffer = Data() + } + private let adbPath: String private let arguments: [String] private let onStdout: @Sendable (Data) -> Void private let onStderr: (@Sendable (String) -> Void)? - - private let process = Process() - private let stdoutPipe = Pipe() - private let stderrPipe = Pipe() + private let state = OSAllocatedUnfairLock(initialState: State()) private let exitSemaphore = DispatchSemaphore(value: 0) - private let lock = NSLock() - private var _stdoutByteCount: Int64 = 0 - private var stderrBuffer = Data() - public init( adbPath: String, arguments: [String], @@ -39,65 +43,60 @@ public final class AdbStreamingProcess: @unchecked Sendable { public func start() throws { let resolvedPath = Adb.resolveOnPATH(adbPath) ?? adbPath - process.executableURL = URL(fileURLWithPath: resolvedPath) - process.arguments = arguments - process.standardOutput = stdoutPipe - process.standardError = stderrPipe - - stdoutPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in - let chunk = handle.availableData - guard let self, !chunk.isEmpty else { return } - self.lock.lock() - self._stdoutByteCount += Int64(chunk.count) - self.lock.unlock() - self.onStdout(chunk) - } - stderrPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in - let chunk = handle.availableData - guard let self, !chunk.isEmpty else { return } - self.lock.lock() - self.stderrBuffer.append(chunk) - self.lock.unlock() - } - - process.terminationHandler = { [exitSemaphore] _ in exitSemaphore.signal() } - - do { - try process.run() - } catch { - let nsErr = error as NSError - let isMissing = - (nsErr.domain == NSCocoaErrorDomain && nsErr.code == 4) || - (nsErr.domain == NSPOSIXErrorDomain && nsErr.code == Int(ENOENT)) - if isMissing { - throw BridgeError.adbMissing + try state.withLock { state in + state.process.executableURL = URL(fileURLWithPath: resolvedPath) + state.process.arguments = arguments + state.process.standardOutput = state.stdoutPipe + state.process.standardError = state.stderrPipe + + state.stdoutPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in + let chunk = handle.availableData + guard let self, !chunk.isEmpty else { return } + self.state.withLock { $0.stdoutByteCount += Int64(chunk.count) } + self.onStdout(chunk) + } + state.stderrPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in + let chunk = handle.availableData + guard let self, !chunk.isEmpty else { return } + self.state.withLock { $0.stderrBuffer.append(chunk) } + } + state.process.terminationHandler = { [exitSemaphore] _ in exitSemaphore.signal() } + + do { + try state.process.run() + } catch { + let nsErr = error as NSError + let isMissing = + (nsErr.domain == NSCocoaErrorDomain && nsErr.code == 4) || + (nsErr.domain == NSPOSIXErrorDomain && nsErr.code == Int(ENOENT)) + if isMissing { + throw BridgeError.adbMissing + } + throw BridgeError.transport(underlying: "Failed to spawn adb: \(error.localizedDescription)", serial: nil) } - throw BridgeError.transport(underlying: "Failed to spawn adb: \(error.localizedDescription)", serial: nil) } } /// Send SIGINT — `screenrecord`'s clean-stop signal (flushes the encoder /// and finalizes its output before exiting). public func interrupt() { - guard process.isRunning else { return } - kill(process.processIdentifier, SIGINT) + state.withLock { state in + if state.process.isRunning { kill(state.process.processIdentifier, SIGINT) } + } } public func terminate() { - guard process.isRunning else { return } - process.terminate() + state.withLock { state in + if state.process.isRunning { state.process.terminate() } + } } - public var isRunning: Bool { process.isRunning } + public var isRunning: Bool { state.withLock { $0.process.isRunning } } - public var stdoutByteCount: Int64 { - lock.lock(); defer { lock.unlock() } - return _stdoutByteCount - } + public var stdoutByteCount: Int64 { state.withLock { $0.stdoutByteCount } } public var collectedStderr: String { - lock.lock(); defer { lock.unlock() } - return String(data: stderrBuffer, encoding: .utf8) ?? "" + state.withLock { String(data: $0.stderrBuffer, encoding: .utf8) ?? "" } } /// Block until the child exits (or `timeout` elapses), then detach the @@ -107,30 +106,26 @@ public final class AdbStreamingProcess: @unchecked Sendable { public func waitForExit(timeout: TimeInterval) -> Int32? { let timedOut = exitSemaphore.wait(timeout: .now() + timeout) == .timedOut if timedOut { - process.terminate() + state.withLock { if $0.process.isRunning { $0.process.terminate() } } _ = exitSemaphore.wait(timeout: .now() + 0.5) } - stdoutPipe.fileHandleForReading.readabilityHandler = nil - stderrPipe.fileHandleForReading.readabilityHandler = nil - - let residual = stdoutPipe.fileHandleForReading.readDataToEndOfFile() - if !residual.isEmpty { - lock.lock() - _stdoutByteCount += Int64(residual.count) - lock.unlock() - onStdout(residual) + let (residualOut, status): (Data, Int32) = state.withLock { state in + state.stdoutPipe.fileHandleForReading.readabilityHandler = nil + state.stderrPipe.fileHandleForReading.readabilityHandler = nil + let residual = state.stdoutPipe.fileHandleForReading.readDataToEndOfFile() + let residualErr = state.stderrPipe.fileHandleForReading.readDataToEndOfFile() + state.stdoutByteCount += Int64(residual.count) + state.stderrBuffer.append(residualErr) + return (residual, state.process.terminationStatus) } - let residualErr = stderrPipe.fileHandleForReading.readDataToEndOfFile() - if !residualErr.isEmpty { - lock.lock() - stderrBuffer.append(residualErr) - lock.unlock() - } - if let onStderr, !collectedStderr.isEmpty { - onStderr(collectedStderr) + + if !residualOut.isEmpty { onStdout(residualOut) } + if let onStderr { + let stderr = collectedStderr + if !stderr.isEmpty { onStderr(stderr) } } - return timedOut ? nil : process.terminationStatus + return timedOut ? nil : status } } diff --git a/Sources/iOSSimBackend/Util/H264MuxingPipeline.swift b/Sources/iOSSimBackend/Util/H264MuxingPipeline.swift index 12d96b6..755dd7f 100644 --- a/Sources/iOSSimBackend/Util/H264MuxingPipeline.swift +++ b/Sources/iOSSimBackend/Util/H264MuxingPipeline.swift @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation +import os import SimUseCore /// Thread-safe bridge from a raw H.264 Annex-B byte stream to an MP4 file, @@ -8,18 +9,22 @@ import SimUseCore /// thread; `ingest(_:)` parses and appends them synchronously so the /// producer sees natural backpressure. A fatal muxer error latches the /// pipeline closed and is reported once through `onFatalError`. -public final class H264MuxingPipeline: @unchecked Sendable { +public final class H264MuxingPipeline: Sendable { + /// Parser + progress state, confined to the lock. The parser is a + /// non-Sendable class touched only under this lock. + private struct State { + var parser = AnnexBStreamParser() + var fatal = false + var closed = false + var framesWritten: Int64 = 0 + var firstFrameReceived = false + var lastIngestHostTime: TimeInterval = 0 + } + private let recorder: H264PassthroughRecorder private let clock: @Sendable () -> TimeInterval private let onFatalError: @Sendable (Error) -> Void - - private let lock = NSLock() - private var parser = AnnexBStreamParser() - private var fatal = false - private var closed = false - private var _framesWritten: Int64 = 0 - private var _firstFrameReceived = false - private var _lastIngestHostTime: TimeInterval = 0 + private let state = OSAllocatedUnfairLock(initialState: State()) public init( recorder: H264PassthroughRecorder, @@ -34,71 +39,58 @@ public final class H264MuxingPipeline: @unchecked Sendable { /// Parse a chunk and append any completed access units. No-op once the /// pipeline has been closed by `finishIngest()` or latched by an error. public func ingest(_ data: Data) { - lock.lock() - defer { lock.unlock() } - guard !fatal, !closed else { return } let now = clock() - _lastIngestHostTime = now - appendLocked(parser.consume(data), hostTime: now) + state.withLock { state in + guard !state.fatal, !state.closed else { return } + state.lastIngestHostTime = now + let accessUnits = state.parser.consume(data) + appendLocked(&state, accessUnits: accessUnits, hostTime: now) + } } /// Start a fresh parser for a new capture segment (Android restarts /// `screenrecord` every 180 s on API < 34). The single monotonic clock /// keeps PTS continuous across the boundary. public func resetParserForNewSegment() { - lock.lock() - defer { lock.unlock() } - parser = AnnexBStreamParser() + state.withLock { $0.parser = AnnexBStreamParser() } } /// Flush the parser's trailing access unit and close the intake. After /// this returns, no further `ingest(_:)` will append, so the caller can /// safely finalize the recorder. Idempotent. public func finishIngest() { - lock.lock() - defer { lock.unlock() } - guard !fatal, !closed else { return } - closed = true - // Stamp the trailing frame with when its data actually arrived, not - // now — the parser holds a frame pending until the next start code, so - // a static screen's sole keyframe arrives at t≈0 but is only flushed - // at stop. Using the flush time would pin the first frame's PTS to the - // stop instant and collapse the clip to zero duration. - let hostTime = _lastIngestHostTime > 0 ? _lastIngestHostTime : clock() - appendLocked(parser.flush(), hostTime: hostTime) + let now = clock() + state.withLock { state in + guard !state.fatal, !state.closed else { return } + state.closed = true + // Stamp the trailing frame with when its data actually arrived, + // not now — the parser holds a frame pending until the next start + // code, so a static screen's sole keyframe arrives at t≈0 but is + // only flushed at stop. Using the flush time would pin the first + // frame's PTS to the stop instant and collapse the clip to zero + // duration. + let hostTime = state.lastIngestHostTime > 0 ? state.lastIngestHostTime : now + let accessUnits = state.parser.flush() + appendLocked(&state, accessUnits: accessUnits, hostTime: hostTime) + } } - private func appendLocked(_ accessUnits: [H264AccessUnit], hostTime: TimeInterval) { - guard !accessUnits.isEmpty, let sps = parser.currentSPS, let pps = parser.currentPPS else { return } + private func appendLocked(_ state: inout State, accessUnits: [H264AccessUnit], hostTime: TimeInterval) { + guard !accessUnits.isEmpty, let sps = state.parser.currentSPS, let pps = state.parser.currentPPS else { return } do { for accessUnit in accessUnits { try recorder.append(accessUnit: accessUnit, sps: sps, pps: pps, hostTime: hostTime) - _framesWritten += 1 - _firstFrameReceived = true + state.framesWritten += 1 + state.firstFrameReceived = true } } catch { - fatal = true + state.fatal = true onFatalError(error) } } - public var framesWritten: Int64 { - lock.lock(); defer { lock.unlock() } - return _framesWritten - } - - public var firstFrameReceived: Bool { - lock.lock(); defer { lock.unlock() } - return _firstFrameReceived - } - - public var lastIngestHostTime: TimeInterval { - lock.lock(); defer { lock.unlock() } - return _lastIngestHostTime - } - - public var hadFatalError: Bool { - lock.lock(); defer { lock.unlock() } - return fatal - } + public var framesWritten: Int64 { state.withLock { $0.framesWritten } } + public var firstFrameReceived: Bool { state.withLock { $0.firstFrameReceived } } + public var lastIngestHostTime: TimeInterval { state.withLock { $0.lastIngestHostTime } } + public var hadFatalError: Bool { state.withLock { $0.fatal } } } diff --git a/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift b/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift index fe0b430..eb765ca 100644 --- a/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift +++ b/Sources/iOSSimBackend/Util/H264PassthroughRecorder.swift @@ -2,6 +2,7 @@ import Foundation import AVFoundation import CoreMedia +import os import SimUseCore public enum H264PassthroughError: Error, LocalizedError, Equatable { @@ -33,63 +34,70 @@ public enum H264PassthroughError: Error, LocalizedError, Equatable { /// AVCC-framed `CMSampleBuffer`s and appended to a passthrough /// `AVAssetWriterInput`. Presentation timestamps come from the host arrival /// clock, since neither capture source embeds timing. -public final class H264PassthroughRecorder: @unchecked Sendable { +public final class H264PassthroughRecorder: Sendable { + /// Mutable + non-Sendable writer state, confined to the lock. + private struct State { + let writer: AVAssetWriter + var input: AVAssetWriterInput? + var formatDescription: CMFormatDescription? + var formatSPS: Data? + var formatPPS: Data? + var formatDimensions: CMVideoDimensions? + var firstHostTime: TimeInterval? + var lastPTS: CMTime = .invalid + var framesAppended: Int64 = 0 + } + private let outputURL: URL - private let writer: AVAssetWriter /// When set, timestamps are laid out as a constant frame rate /// (`frameIndex / frameRate`) instead of derived from host arrival time. /// This removes the byte-arrival jitter of an eager fixed-rate stream; /// nil keeps the variable-rate host-clock behavior (Android screenrecord). private let frameRate: Int? - private var input: AVAssetWriterInput? - - private var formatDescription: CMFormatDescription? - private var formatSPS: Data? - private var formatPPS: Data? - private var formatDimensions: CMVideoDimensions? + private let state: OSAllocatedUnfairLock - private var firstHostTime: TimeInterval? - private var lastPTS: CMTime = .invalid - - public private(set) var framesAppended: Int64 = 0 + public var framesAppended: Int64 { state.withLock { $0.framesAppended } } public init(outputURL: URL, frameRate: Int? = nil) throws { self.outputURL = outputURL self.frameRate = frameRate - self.writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + self.state = OSAllocatedUnfairLock(initialState: State(writer: writer)) } /// Append one access unit. The first call lazily builds the format /// description and starts the writer session (parameter sets are only /// known once the first SPS/PPS arrive). public func append(accessUnit: H264AccessUnit, sps: Data, pps: Data, hostTime: TimeInterval) throws { - try ensureStarted(sps: sps, pps: pps, hostTime: hostTime) - guard let input, let formatDescription, let firstHostTime else { - throw H264PassthroughError.appendFailed("writer not initialized") - } + try state.withLock { state in + try ensureStarted(&state, sps: sps, pps: pps, hostTime: hostTime) + guard let input = state.input, let formatDescription = state.formatDescription, let firstHostTime = state.firstHostTime else { + throw H264PassthroughError.appendFailed("writer not initialized") + } - try H264StreamRecorder.waitUntilReady( - isReady: { input.isReadyForMoreMediaData }, - timeout: H264StreamRecorder.readinessTimeout - ) + try H264StreamRecorder.waitUntilReady( + isReady: { input.isReadyForMoreMediaData }, + timeout: H264StreamRecorder.readinessTimeout + ) - let pts: CMTime - if let frameRate { - pts = CMTime(value: framesAppended, timescale: CMTimeScale(frameRate)) - } else { - pts = Self.normalizedPTS(hostTime: hostTime, firstHostTime: firstHostTime, lastPTS: lastPTS) - } - let sampleBuffer = try Self.makeSampleBuffer( - avcc: Self.avccData(for: accessUnit), - formatDescription: formatDescription, - pts: pts, - isIDR: accessUnit.isIDR - ) - guard input.append(sampleBuffer) else { - throw H264PassthroughError.appendFailed(writer.error?.localizedDescription ?? "unknown writer error") + let pts: CMTime + if let frameRate { + pts = CMTime(value: state.framesAppended, timescale: CMTimeScale(frameRate)) + } else { + pts = Self.normalizedPTS(hostTime: hostTime, firstHostTime: firstHostTime, lastPTS: state.lastPTS) + } + let sampleBuffer = try Self.makeSampleBuffer( + avcc: Self.avccData(for: accessUnit), + formatDescription: formatDescription, + pts: pts, + isIDR: accessUnit.isIDR + ) + guard input.append(sampleBuffer) else { + throw H264PassthroughError.appendFailed(state.writer.error?.localizedDescription ?? "unknown writer error") + } + state.lastPTS = pts + state.framesAppended += 1 } - lastPTS = pts - framesAppended += 1 } /// Finalize the MP4. `stopHostTime` sets the session end so the final @@ -97,80 +105,84 @@ public final class H264PassthroughRecorder: @unchecked Sendable { /// Throws `.noFramesCaptured` — after deleting the empty output — when /// no frame was ever appended. public func finish(stopHostTime: TimeInterval?) async throws { - guard let input, framesAppended > 0, let firstHostTime else { - if writer.status == .writing { - writer.cancelWriting() - } + let hasFrames = state.withLock { $0.input != nil && $0.framesAppended > 0 } + guard hasFrames else { + state.withLock { if $0.writer.status == .writing { $0.writer.cancelWriting() } } try? FileManager.default.removeItem(at: outputURL) throw H264PassthroughError.noFramesCaptured } - if let frameRate { - // Hold the last frame for one frame interval past its PTS. - writer.endSession(atSourceTime: CMTime(value: framesAppended, timescale: CMTimeScale(frameRate))) - } else if let stopHostTime { - let endPTS = Self.normalizedPTS(hostTime: stopHostTime, firstHostTime: firstHostTime, lastPTS: lastPTS) - writer.endSession(atSourceTime: endPTS) - } - input.markAsFinished() try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - writer.finishWriting { - if let error = self.writer.error { - continuation.resume(throwing: error) - } else { - continuation.resume(returning: ()) + state.withLock { state in + if let frameRate { + // Hold the last frame for one frame interval past its PTS. + state.writer.endSession(atSourceTime: CMTime(value: state.framesAppended, timescale: CMTimeScale(frameRate))) + } else if let stopHostTime, let firstHostTime = state.firstHostTime { + let endPTS = Self.normalizedPTS(hostTime: stopHostTime, firstHostTime: firstHostTime, lastPTS: state.lastPTS) + state.writer.endSession(atSourceTime: endPTS) + } + state.input?.markAsFinished() + state.writer.finishWriting { + let error = self.state.withLock { $0.writer.error } + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: ()) + } } } } } public func invalidate() { - if writer.status == .writing { - input?.markAsFinished() - writer.cancelWriting() + state.withLock { state in + if state.writer.status == .writing { + state.input?.markAsFinished() + state.writer.cancelWriting() + } } } // MARK: - Startup / segment handling - private func ensureStarted(sps: Data, pps: Data, hostTime: TimeInterval) throws { - guard input == nil else { + private func ensureStarted(_ state: inout State, sps: Data, pps: Data, hostTime: TimeInterval) throws { + guard state.input == nil else { // Already recording: only react if the parameter sets changed // (new segment on Android's 180 s restart, or a rotation). - guard sps != formatSPS || pps != formatPPS else { return } + guard sps != state.formatSPS || pps != state.formatPPS else { return } let newFormat = try Self.makeFormatDescription(sps: sps, pps: pps) let newDimensions = CMVideoFormatDescriptionGetDimensions(newFormat) - if let old = formatDimensions, old.width != newDimensions.width || old.height != newDimensions.height { + if let old = state.formatDimensions, old.width != newDimensions.width || old.height != newDimensions.height { throw H264PassthroughError.dimensionsChanged( old: "\(old.width)x\(old.height)", new: "\(newDimensions.width)x\(newDimensions.height)" ) } FileHandle.standardError.write(Data("note: stream parameter sets changed mid-recording (same size); continuing\n".utf8)) - formatDescription = newFormat - formatSPS = sps - formatPPS = pps + state.formatDescription = newFormat + state.formatSPS = sps + state.formatPPS = pps return } let format = try Self.makeFormatDescription(sps: sps, pps: pps) let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: nil, sourceFormatHint: format) writerInput.expectsMediaDataInRealTime = true - guard writer.canAdd(writerInput) else { + guard state.writer.canAdd(writerInput) else { throw H264PassthroughError.appendFailed("writer cannot add passthrough input") } - writer.add(writerInput) - guard writer.startWriting() else { - throw H264PassthroughError.appendFailed(writer.error?.localizedDescription ?? "startWriting failed") + state.writer.add(writerInput) + guard state.writer.startWriting() else { + throw H264PassthroughError.appendFailed(state.writer.error?.localizedDescription ?? "startWriting failed") } - writer.startSession(atSourceTime: .zero) + state.writer.startSession(atSourceTime: .zero) - input = writerInput - formatDescription = format - formatSPS = sps - formatPPS = pps - formatDimensions = CMVideoFormatDescriptionGetDimensions(format) - firstHostTime = hostTime + state.input = writerInput + state.formatDescription = format + state.formatSPS = sps + state.formatPPS = pps + state.formatDimensions = CMVideoFormatDescriptionGetDimensions(format) + state.firstHostTime = hostTime } // MARK: - Pure helpers (unit-tested) diff --git a/Sources/iOSSimBackend/Util/VideoCommandSupport.swift b/Sources/iOSSimBackend/Util/VideoCommandSupport.swift index 86ca1f0..d25b46f 100644 --- a/Sources/iOSSimBackend/Util/VideoCommandSupport.swift +++ b/Sources/iOSSimBackend/Util/VideoCommandSupport.swift @@ -4,6 +4,7 @@ import FBSimulatorControl @preconcurrency import FBControlCore import AVFoundation import ImageIO +import os #if os(macOS) import AppKit import SimUseCore @@ -15,43 +16,37 @@ import SimUseCore /// that follows SIGTERM with a short-grace SIGKILL must reach /// `recorder.finish()` before the kill lands, otherwise the mp4 trailer /// (moov atom) is never written. An actor-backed flag adds ~10-100 ms of -/// scheduler jitter on every cancel/check; using `NSLock` keeps the +/// scheduler jitter on every cancel/check; `OSAllocatedUnfairLock` keeps the /// handler and loop pickup synchronous. -public final class CancellationFlag: @unchecked Sendable { - private let lock = NSLock() - private var _value = false +public final class CancellationFlag: Sendable { + private let value = OSAllocatedUnfairLock(initialState: false) public init() {} public func cancel() { - lock.lock() - defer { lock.unlock() } - _value = true + value.withLock { $0 = true } } public func isCancelled() -> Bool { - lock.lock() - defer { lock.unlock() } - return _value + value.withLock { $0 } } } /// A latch that transitions to "set" exactly once. Guarantees a /// `CheckedContinuation` is resumed a single time when two callbacks race /// (a completion handler versus a timeout). -public final class OnceFlag: @unchecked Sendable { - private let lock = NSLock() - private var fired = false +public final class OnceFlag: Sendable { + private let fired = OSAllocatedUnfairLock(initialState: false) public init() {} /// Returns true the first time it is called, false thereafter. public func trySet() -> Bool { - lock.lock() - defer { lock.unlock() } - guard !fired else { return false } - fired = true - return true + fired.withLock { fired in + guard !fired else { return false } + fired = true + return true + } } } @@ -62,26 +57,19 @@ public final class OnceFlag: @unchecked Sendable { /// the stream runs until a signal arrives. The command loop polls this /// box instead, so the first failure terminates streaming and surfaces /// as a thrown error rather than being lost to stderr. -public final class FirstErrorBox: @unchecked Sendable { - private let lock = NSLock() - private var _error: Error? +public final class FirstErrorBox: Sendable { + private let stored = OSAllocatedUnfairLock(initialState: nil) public init() {} /// Records `error` unless one is already recorded; later calls are no-ops. public func set(_ error: Error) { - lock.lock() - defer { lock.unlock() } - if _error == nil { - _error = error - } + stored.withLock { if $0 == nil { $0 = error } } } /// The first error recorded, or nil if none has been. public var first: Error? { - lock.lock() - defer { lock.unlock() } - return _error + stored.withLock { $0 } } } @@ -264,10 +252,14 @@ public struct VideoFrameUtilities { } } -public final class H264StreamRecorder: @unchecked Sendable { - private let writer: AVAssetWriter - private let input: AVAssetWriterInput - private let adaptor: AVAssetWriterInputPixelBufferAdaptor +public final class H264StreamRecorder: Sendable { + /// The non-Sendable AVFoundation objects, confined to the lock. + private struct State { + let writer: AVAssetWriter + let input: AVAssetWriterInput + let adaptor: AVAssetWriterInputPixelBufferAdaptor + } + private let state: OSAllocatedUnfairLock private let width: Int private let height: Int @@ -316,9 +308,7 @@ public final class H264StreamRecorder: @unchecked Sendable { } writer.startSession(atSourceTime: .zero) - self.writer = writer - self.input = input - self.adaptor = adaptor + self.state = OSAllocatedUnfairLock(initialState: State(writer: writer, input: input, adaptor: adaptor)) } /// How long `append` waits for the writer input to drain before @@ -348,55 +338,62 @@ public final class H264StreamRecorder: @unchecked Sendable { } public func append(image: CGImage, presentationTime: CMTime) throws { - try Self.waitUntilReady( - isReady: { input.isReadyForMoreMediaData }, - timeout: Self.readinessTimeout - ) - - guard let pixelBuffer = Self.makePixelBuffer(width: width, height: height, adaptor: adaptor) else { - throw VideoProcessingError.failedToAllocatePixelBuffer - } + try state.withLock { state in + try Self.waitUntilReady( + isReady: { state.input.isReadyForMoreMediaData }, + timeout: Self.readinessTimeout + ) + + guard let pixelBuffer = Self.makePixelBuffer(width: width, height: height, adaptor: state.adaptor) else { + throw VideoProcessingError.failedToAllocatePixelBuffer + } - CVPixelBufferLockBaseAddress(pixelBuffer, []) - defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } - - guard let context = CGContext( - data: CVPixelBufferGetBaseAddress(pixelBuffer), - width: width, - height: height, - bitsPerComponent: 8, - bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), - space: CGColorSpaceCreateDeviceRGB(), - bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue - ) else { - throw CLIError(errorDescription: "Failed to create drawing context") - } + CVPixelBufferLockBaseAddress(pixelBuffer, []) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } + + guard let context = CGContext( + data: CVPixelBufferGetBaseAddress(pixelBuffer), + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue + ) else { + throw CLIError(errorDescription: "Failed to create drawing context") + } - context.interpolationQuality = .high - context.draw(image, in: CGRect(x: 0, y: CGFloat(height), width: CGFloat(width), height: -CGFloat(height))) + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: CGFloat(height), width: CGFloat(width), height: -CGFloat(height))) - guard adaptor.append(pixelBuffer, withPresentationTime: presentationTime) else { - throw CLIError(errorDescription: "Failed to append frame: \(writer.error?.localizedDescription ?? "Unknown error")") + guard state.adaptor.append(pixelBuffer, withPresentationTime: presentationTime) else { + throw CLIError(errorDescription: "Failed to append frame: \(state.writer.error?.localizedDescription ?? "Unknown error")") + } } } public func finish() async throws { - input.markAsFinished() try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - writer.finishWriting { - if let error = self.writer.error { - continuation.resume(throwing: error) - } else { - continuation.resume(returning: ()) + state.withLock { state in + state.input.markAsFinished() + state.writer.finishWriting { + let error = self.state.withLock { $0.writer.error } + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: ()) + } } } } } public func invalidate() { - if writer.status == .writing { - input.markAsFinished() - writer.cancelWriting() + state.withLock { state in + if state.writer.status == .writing { + state.input.markAsFinished() + state.writer.cancelWriting() + } } } diff --git a/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift b/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift index 7d23c03..2117ecb 100644 --- a/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift +++ b/Sources/iOSSimBackend/Verbs/IOSSimRecordVideoCommand.swift @@ -220,11 +220,21 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { } }) - // Give the stream a moment to fail fast on an attach error before we - // commit to it — a failure here means H.264 streaming is unavailable. - try await Task.sleep(nanoseconds: 1_000_000_000) - if let error = streamError.first, !pipeline.firstFrameReceived { - throw StreamUnavailableError(underlying: (error as NSError).localizedDescription) + // Wait for the stream to actually start delivering before committing + // to it. FBSimulatorVideoStream occasionally attaches cleanly yet + // pushes no frames (cold start / wedged encoder); treating that as + // "unavailable" falls back to screenshot capture rather than producing + // an empty recording. + let firstFrameDeadline = Date().addingTimeInterval(2.0) + while !pipeline.firstFrameReceived { + if Task.isCancelled || cancellationFlag.isCancelled() { break } + if let error = streamError.first { + throw StreamUnavailableError(underlying: (error as NSError).localizedDescription) + } + if Date() >= firstFrameDeadline { + throw StreamUnavailableError(underlying: "no frames received within 2s of stream start") + } + try? await cancellableSleep(seconds: 0.05, flag: cancellationFlag) } try await runStreamPollLoop(pipeline: pipeline, streamError: streamError, cancellationFlag: cancellationFlag) @@ -252,19 +262,12 @@ public struct IOSSimRecordVideoCommand: SimUseExecutableCommand { streamError: FirstErrorBox, cancellationFlag: CancellationFlag ) async throws { - let startTime = Date() - var lastProgress = startTime - var warnedNoFrames = false - + var lastProgress = Date() while true { if Task.isCancelled || cancellationFlag.isCancelled() { break } if streamError.first != nil { break } let now = Date() - if !warnedNoFrames, !pipeline.firstFrameReceived, now.timeIntervalSince(startTime) > 3 { - warnedNoFrames = true - FileHandle.standardError.write(Data("warning: no video frames received yet\n".utf8)) - } if now.timeIntervalSince(lastProgress) >= 2 { lastProgress = now FileHandle.standardError.write(Data(String(format: "Captured %lld frames\n", pipeline.framesWritten).utf8)) diff --git a/Tests/H264MuxingPipelineTests.swift b/Tests/H264MuxingPipelineTests.swift index 2aaef07..fa191fd 100644 --- a/Tests/H264MuxingPipelineTests.swift +++ b/Tests/H264MuxingPipelineTests.swift @@ -2,6 +2,7 @@ import Testing import Foundation import AVFoundation +import os @testable import iOSSimBackend @Suite("H264MuxingPipeline chunked ingest") @@ -12,14 +13,13 @@ struct H264MuxingPipelineTests { } /// Deterministic 10 fps clock so PTS assertions don't depend on wall time. - private final class FakeClock: @unchecked Sendable { - private let lock = NSLock() - private var t = 500.0 + private final class FakeClock: Sendable { + private let time = OSAllocatedUnfairLock(initialState: 500.0) func tick() -> TimeInterval { - lock.lock(); defer { lock.unlock() } - let value = t - t += 0.1 - return value + time.withLock { time in + defer { time += 0.1 } + return time + } } } diff --git a/Tests/RecordVideoTests.swift b/Tests/RecordVideoTests.swift index da7c529..7380fb7 100644 --- a/Tests/RecordVideoTests.swift +++ b/Tests/RecordVideoTests.swift @@ -119,7 +119,9 @@ struct RecordVideoTests { let sentinel = tempDir.appendingPathComponent("sentinel.txt") try "sentinel".write(to: sentinel, atomically: true, encoding: .utf8) - let result = try await invokeRecordVideo(duration: 1.0, outputPath: tempDir.path) + // 3 s, not 1 s: this exercises directory handling, and the H.264 + // stream needs a beat to attach and deliver its first frame. + let result = try await invokeRecordVideo(duration: 3.0, outputPath: tempDir.path) #expect(FileManager.default.fileExists(atPath: sentinel.path)) #expect(result.exitCode == 0)