Tune Studio expose for realtime H264

DjDeveloperr · DjDeveloperr · commit 9864f6210c10 · 2026-04-30T21:39:30.000-04:00
diff --git a/README.md b/README.md
@@ -78,8 +78,9 @@ simdeck studio expose "iPhone 17 Pro"
 The command starts or reuses the local daemon, creates an ephemeral Studio
 session, prints a unique `https://simdeck.djdev.me/simulator/...` URL, and keeps
 the outbound bridge alive until you press Ctrl-C. It uses hardware H.264 by
-default; pass `--low-latency` to switch to software H.264's low-latency profile
-for slower Macs or shared runners.
+default with realtime stream settings for remote viewing; pass `--low-latency`
+to switch to software H.264's low-latency profile for slower Macs or shared
+runners.
 
 CLI commands automatically use the same warm daemon:
 
diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m
@@ -8,16 +8,19 @@
 #include <stdlib.h>
 
 static const int32_t XCWMaximumEncodedDimension = 1920;
+static const int32_t XCWMaximumRealtimeHardwareEncodedDimension = 1600;
 static const int32_t XCWMaximumSoftwareEncodedDimension = 1600;
 static const int32_t XCWMaximumLowLatencySoftwareEncodedDimension = 1170;
 static const int32_t XCWTargetRealTimeFrameRate = 60;
 static const int32_t XCWTargetSoftwareFrameRate = 60;
 static const int32_t XCWTargetLowLatencySoftwareFrameRate = 15;
 static const NSUInteger XCWMaximumInFlightFrames = 2;
 static const int32_t XCWMinimumAverageBitRate = 18000000;
+static const int32_t XCWMinimumRealtimeAverageBitRate = 5000000;
 static const int32_t XCWMinimumSoftwareAverageBitRate = 3000000;
 static const int32_t XCWMinimumLowLatencySoftwareAverageBitRate = 2000000;
 static const int64_t XCWBitsPerPixelBudget = 10;
+static const int64_t XCWRealtimeBitsPerPixelBudget = 5;
 static const int64_t XCWSoftwareBitsPerPixelBudget = 6;
 static const int64_t XCWLowLatencySoftwareBitsPerPixelBudget = 3;
 static const uint64_t XCWSoftwareMinimumFrameIntervalUs = 16667;
@@ -57,6 +60,15 @@ static BOOL XCWLowLatencyModeFromEnvironment(void) {
         [value isEqualToString:@"on"];
 }
 
+static BOOL XCWRealtimeStreamModeFromEnvironment(void) {
+    const char *rawValue = getenv("SIMDECK_REALTIME_STREAM");
+    NSString *value = rawValue != NULL ? [[[NSString alloc] initWithUTF8String:rawValue] lowercaseString] : @"";
+    return [value isEqualToString:@"1"] ||
+        [value isEqualToString:@"true"] ||
+        [value isEqualToString:@"yes"] ||
+        [value isEqualToString:@"on"];
+}
+
 static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) {
     switch (mode) {
         case XCWVideoEncoderModeH264Hardware:
@@ -206,13 +218,15 @@ static int32_t XCWRoundToEvenDimension(double value) {
     return rounded;
 }
 
-static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode) {
+static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode, BOOL realtimeStreamMode) {
     if (width <= 0 || height <= 0) {
         return CGSizeZero;
     }
 
     int32_t maximumDimension = XCWMaximumEncodedDimension;
-    if (mode == XCWVideoEncoderModeH264Software) {
+    if (mode == XCWVideoEncoderModeH264Hardware && realtimeStreamMode) {
+        maximumDimension = XCWMaximumRealtimeHardwareEncodedDimension;
+    } else if (mode == XCWVideoEncoderModeH264Software) {
         maximumDimension = lowLatencyMode
             ? XCWMaximumLowLatencySoftwareEncodedDimension
             : XCWMaximumSoftwareEncodedDimension;
@@ -227,10 +241,13 @@ static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XC
                       XCWRoundToEvenDimension(height * scale));
 }
 
-static int32_t XCWAverageBitRateForDimensions(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode) {
+static int32_t XCWAverageBitRateForDimensions(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode, BOOL realtimeStreamMode) {
     int64_t bitsPerPixelBudget = XCWBitsPerPixelBudget;
     int64_t minimumAverageBitRate = XCWMinimumAverageBitRate;
-    if (mode == XCWVideoEncoderModeH264Software) {
+    if (mode == XCWVideoEncoderModeH264Hardware && realtimeStreamMode) {
+        bitsPerPixelBudget = XCWRealtimeBitsPerPixelBudget;
+        minimumAverageBitRate = XCWMinimumRealtimeAverageBitRate;
+    } else if (mode == XCWVideoEncoderModeH264Software) {
         bitsPerPixelBudget = lowLatencyMode
             ? XCWLowLatencySoftwareBitsPerPixelBudget
             : XCWSoftwareBitsPerPixelBudget;
@@ -352,6 +369,7 @@ @implementation XCWH264Encoder {
     OSType _scaledPixelFormat;
     XCWVideoEncoderMode _encoderMode;
     BOOL _lowLatencyMode;
+    BOOL _realtimeStreamMode;
     CMVideoCodecType _codecType;
     BOOL _hardwareAccelerated;
     NSUInteger _inputFrameCount;
@@ -386,6 +404,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler
     _needsKeyFrame = YES;
     _encoderMode = XCWVideoEncoderModeFromEnvironment();
     _lowLatencyMode = (_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment();
+    _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode;
     _codecType = XCWVideoCodecTypeForMode(_encoderMode);
     _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
     return self;
@@ -456,6 +475,7 @@ - (NSDictionary *)statsRepresentation {
             @"transportCodec": XCWCodecName(self->_codecType),
             @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode),
             @"lowLatencyMode": @(self->_lowLatencyMode),
+            @"realtimeStreamMode": @(self->_realtimeStreamMode),
             @"encoderId": XCWVideoEncoderIDForMode(self->_encoderMode) ?: @"automatic",
             @"hardwareAccelerated": @(self->_hardwareAccelerated),
             @"lastSessionStatus": @(self->_lastSessionStatus),
@@ -483,7 +503,7 @@ - (void)invalidate {
 }
 
 - (NSUInteger)maximumInFlightFrameCountLocked {
-    return (_encoderMode == XCWVideoEncoderModeH264Software && _lowLatencyMode) ? 1 : XCWMaximumInFlightFrames;
+    return (_realtimeStreamMode || (_encoderMode == XCWVideoEncoderModeH264Software && _lowLatencyMode)) ? 1 : XCWMaximumInFlightFrames;
 }
 
 - (uint64_t)minimumSoftwareFrameIntervalUsLocked {
@@ -600,7 +620,7 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
         return NO;
     }
 
-    CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _encoderMode, _lowLatencyMode);
+    CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _encoderMode, _lowLatencyMode, _realtimeStreamMode);
     int32_t targetWidth = (int32_t)targetSize.width;
     int32_t targetHeight = (int32_t)targetSize.height;
     if (targetWidth <= 0 || targetHeight <= 0) {
@@ -708,7 +728,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height
     _needsKeyFrame = YES;
 
     int expectedFrameRate = [self expectedFrameRateLocked];
-    int averageBitRate = XCWAverageBitRateForDimensions(width, height, _encoderMode, _lowLatencyMode);
+    int averageBitRate = XCWAverageBitRateForDimensions(width, height, _encoderMode, _lowLatencyMode, _realtimeStreamMode);
 
     VTSessionSetProperty(session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
     if (@available(macOS 10.14, *)) {
@@ -733,8 +753,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height
         VTSessionSetProperty(session, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel);
     }
     VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate));
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(_lowLatencyMode ? expectedFrameRate : expectedFrameRate * 2));
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(_lowLatencyMode ? 1.0 : 2.0));
+    BOOL shortKeyframeInterval = _lowLatencyMode || _realtimeStreamMode;
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? expectedFrameRate : expectedFrameRate * 2));
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 1.0 : 2.0));
     VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate));
     if (@available(macOS 11.0, *)) {
         VTSessionSetProperty(session,
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
@@ -476,6 +476,7 @@ async fn health(State(state): State<AppState>) -> Json<Value> {
         "timestamp": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO).as_secs_f64(),
         "videoCodec": active_video_codec(&state.config),
         "lowLatency": state.config.low_latency,
+        "realtimeStream": crate::transport::webrtc::realtime_stream_enabled(),
         "webRtc": {
             "iceServers": crate::transport::webrtc::client_ice_servers(),
             "iceTransportPolicy": crate::transport::webrtc::ice_transport_policy_label()
diff --git a/server/src/main.rs b/server/src/main.rs
@@ -532,6 +532,12 @@ struct DaemonMetadata {
     started_at: u64,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     log_path: Option<PathBuf>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    video_codec: Option<String>,
+    #[serde(default)]
+    low_latency: bool,
+    #[serde(default)]
+    realtime_stream: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -542,6 +548,7 @@ struct DaemonLaunchOptions {
     client_root: Option<PathBuf>,
     video_codec: VideoCodecMode,
     low_latency: bool,
+    realtime_stream: bool,
 }
 
 struct StudioExposeOptions {
@@ -581,6 +588,7 @@ impl Default for DaemonLaunchOptions {
             client_root: None,
             video_codec: VideoCodecMode::H264Software,
             low_latency: false,
+            realtime_stream: false,
         }
     }
 }
@@ -593,9 +601,10 @@ fn ensure_project_daemon_with_status(
     options: DaemonLaunchOptions,
 ) -> anyhow::Result<(DaemonMetadata, bool)> {
     if let Some(metadata) = read_daemon_metadata().ok().flatten() {
-        if daemon_is_healthy(&metadata) {
+        if daemon_is_healthy(&metadata) && daemon_matches_launch_options(&metadata, &options) {
             return Ok((metadata, false));
         }
+        let _ = terminate_daemon_metadata(&metadata);
     }
     Ok((start_project_daemon(options)?, true))
 }
@@ -666,17 +675,21 @@ done
         recoverable_restart_exit_code = RECOVERABLE_RESTART_EXIT_CODE
     );
 
-    let child = ProcessCommand::new("/bin/sh")
+    let mut command = ProcessCommand::new("/bin/sh");
+    command
         .arg("-c")
         .arg(supervisor_script)
         .arg("simdeck-supervisor")
         .arg(&executable)
         .args(args)
+        .env(
+            "SIMDECK_REALTIME_STREAM",
+            if options.realtime_stream { "1" } else { "0" },
+        )
         .stdin(Stdio::null())
         .stdout(Stdio::from(log_stdout))
-        .stderr(Stdio::from(log_stderr))
-        .spawn()
-        .context("start project SimDeck daemon")?;
+        .stderr(Stdio::from(log_stderr));
+    let child = command.spawn().context("start project SimDeck daemon")?;
 
     let metadata = DaemonMetadata {
         project_root,
@@ -687,6 +700,9 @@ done
         binary_path: executable,
         started_at: now_secs(),
         log_path: Some(log_path),
+        video_codec: Some(options.video_codec.as_env_value().to_owned()),
+        low_latency: options.low_latency,
+        realtime_stream: options.realtime_stream,
     };
     write_daemon_metadata(&metadata)?;
     wait_for_daemon(&metadata, Duration::from_secs(15))?;
@@ -820,6 +836,15 @@ fn daemon_is_healthy(metadata: &DaemonMetadata) -> bool {
     http_get_json(&metadata.http_url, "/api/health").is_ok()
 }
 
+fn daemon_matches_launch_options(metadata: &DaemonMetadata, options: &DaemonLaunchOptions) -> bool {
+    metadata
+        .video_codec
+        .as_deref()
+        .is_some_and(|codec| codec == options.video_codec.as_env_value())
+        && metadata.low_latency == options.low_latency
+        && metadata.realtime_stream == options.realtime_stream
+}
+
 fn read_daemon_metadata() -> anyhow::Result<Option<DaemonMetadata>> {
     let path = daemon_metadata_path()?;
     if !path.exists() {
@@ -1024,6 +1049,9 @@ fn run_foreground_ui(selector: Option<String>) -> anyhow::Result<()> {
         binary_path: executable,
         started_at: now_secs(),
         log_path: None,
+        video_codec: Some(video_codec.as_env_value().to_owned()),
+        low_latency,
+        realtime_stream: false,
     };
     write_daemon_metadata(&metadata)?;
 
@@ -1089,6 +1117,7 @@ fn expose_to_studio(options: StudioExposeOptions) -> anyhow::Result<()> {
         client_root: None,
         video_codec: options.video_codec,
         low_latency: options.low_latency,
+        realtime_stream: true,
     })?;
     let selected = if let Some(selector) = options.simulator.as_deref() {
         select_studio_simulator(&metadata.http_url, selector)
@@ -1276,6 +1305,7 @@ fn main() -> anyhow::Result<()> {
                 client_root,
                 video_codec,
                 low_latency,
+                realtime_stream: false,
             })?;
             if open {
                 open_browser(&metadata.http_url)?;
@@ -1299,6 +1329,7 @@ fn main() -> anyhow::Result<()> {
                     client_root,
                     video_codec,
                     low_latency,
+                    realtime_stream: false,
                 })?;
                 print_daemon_start_result(&metadata, started)
             }
@@ -1316,6 +1347,7 @@ fn main() -> anyhow::Result<()> {
                 client_root,
                 video_codec,
                 low_latency,
+                realtime_stream: false,
             }),
             DaemonCommand::Stop => stop_project_daemon(),
             DaemonCommand::Killall => kill_all_project_daemons(),
@@ -1345,6 +1377,10 @@ fn main() -> anyhow::Result<()> {
                     binary_path: env::current_exe().context("resolve daemon executable")?,
                     started_at: now_secs(),
                     log_path,
+                    video_codec: Some(video_codec.as_env_value().to_owned()),
+                    low_latency,
+                    realtime_stream: crate::transport::webrtc::realtime_stream_enabled()
+                        || low_latency,
                 })?;
                 let result = serve_with_appkit(
                     port,
@@ -2144,6 +2180,15 @@ fn serve_with_appkit(
 ) -> anyhow::Result<()> {
     std::env::set_var("SIMDECK_VIDEO_CODEC", video_codec.as_env_value());
     std::env::set_var("SIMDECK_LOW_LATENCY", if low_latency { "1" } else { "0" });
+    let inherited_realtime_stream = crate::transport::webrtc::realtime_stream_enabled();
+    std::env::set_var(
+        "SIMDECK_REALTIME_STREAM",
+        if inherited_realtime_stream || low_latency {
+            "1"
+        } else {
+            "0"
+        },
+    );
     std::env::set_var(RESTART_ON_CORE_SIMULATOR_MISMATCH_ENV, "1");
     start_fd_pressure_watchdog();
     unsafe {
diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs
@@ -38,6 +38,7 @@ const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(100);
 const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67);
 const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250);
 const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120);
+const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45);
 static WEBRTC_MEDIA_STREAMS: OnceLock<Mutex<HashMap<String, Vec<broadcast::Sender<()>>>>> =
     OnceLock::new();
 
@@ -751,17 +752,28 @@ async fn write_frame_sample_with_timeout(
     frame: &crate::transport::packet::SharedFrame,
     duration: Duration,
 ) -> anyhow::Result<bool> {
-    match time::timeout(
-        WEBRTC_WRITE_TIMEOUT,
-        write_frame_sample(video_track, frame, duration),
-    )
-    .await
-    {
+    let timeout = if realtime_stream_enabled() {
+        WEBRTC_REALTIME_WRITE_TIMEOUT
+    } else {
+        WEBRTC_WRITE_TIMEOUT
+    };
+    match time::timeout(timeout, write_frame_sample(video_track, frame, duration)).await {
         Ok(result) => result.map(|()| true),
         Err(_) => Ok(false),
     }
 }
 
+pub fn realtime_stream_enabled() -> bool {
+    std::env::var("SIMDECK_REALTIME_STREAM")
+        .ok()
+        .is_some_and(|value| {
+            matches!(
+                value.trim().to_ascii_lowercase().as_str(),
+                "1" | "true" | "yes" | "on"
+            )
+        })
+}
+
 fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result<Vec<u8>> {
     let data = frame.data.as_ref();
     let description = frame.description.as_ref().map(bytes::Bytes::as_ref);
diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md
@@ -46,7 +46,9 @@ directly with the runner.
 For an ad-hoc local provider that can be opened from another browser or phone,
 run `simdeck studio expose "iPhone 17 Pro"` and keep that process running. It
 prints the unique Studio simulator URL. This defaults to hardware H.264; add
-`--low-latency` to use software H.264's low-latency profile.
+`--low-latency` to use software H.264's low-latency profile. Studio expose uses
+realtime stream settings so remote viewers drop stale frames instead of building
+latency.
 
 The local viewer gets the API token automatically. LAN browsers pair with the printed code before receiving the API cookie. Direct HTTP calls need `X-SimDeck-Token` or `Authorization: Bearer <token>`.
 
