Fallback to software encoder when web client is backgrounded

DjDeveloperr · DjDeveloperr · commit 979a37016fc3 · 2026-05-12T18:27:16.000-04:00
diff --git a/cli/XCWH264Encoder.h b/cli/XCWH264Encoder.h
@@ -20,6 +20,7 @@ typedef void (^XCWH264EncoderOutputHandler)(NSData *sampleData,
 - (void)encodePixelBuffer:(CVPixelBufferRef)pixelBuffer;
 - (void)requestKeyFrame;
 - (void)reconfigureForStreamQualityChange;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)statsRepresentation;
 - (void)invalidate;
 
diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m
@@ -509,6 +509,7 @@ @implementation XCWH264Encoder {
     BOOL _scalingActive;
     XCWVideoEncoderMode _encoderMode;
     XCWVideoEncoderMode _activeEncoderMode;
+    BOOL _clientForeground;
     BOOL _lowLatencyMode;
     BOOL _realtimeStreamMode;
     CMVideoCodecType _codecType;
@@ -559,6 +560,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler
     _needsKeyFrame = YES;
     _encoderMode = XCWVideoEncoderModeFromEnvironment();
     _activeEncoderMode = _encoderMode;
+    _clientForeground = YES;
     _lowLatencyMode = (_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment();
     _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode;
     _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode);
@@ -621,6 +623,17 @@ - (void)reconfigureForStreamQualityChange {
         self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
         self->_hardwarePacedFrameCount = 0;
         self->_autoSoftwareFallbackUntilUs = 0;
+        [self updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)(CACurrentMediaTime() * 1000000.0)];
+    });
+}
+
+- (void)setClientForeground:(BOOL)foreground {
+    dispatch_async(_queue, ^{
+        if (self->_clientForeground == foreground) {
+            return;
+        }
+        self->_clientForeground = foreground;
+        [self updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)(CACurrentMediaTime() * 1000000.0)];
     });
 }
 
@@ -699,6 +712,7 @@ - (NSDictionary *)statsRepresentation {
             @"transportCodec": XCWCodecName(self->_codecType),
             @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode),
             @"activeEncoderMode": XCWVideoEncoderModeName(self->_activeEncoderMode),
+            @"clientForeground": @(self->_clientForeground),
             @"autoSoftwareFallbackActive": @(autoSoftwareFallbackActive),
             @"autoSoftwareFallbackRemainingUs": @(autoSoftwareFallbackRemainingUs),
             @"autoSoftwareFallbacks": @(self->_autoSoftwareFallbackCount),
@@ -781,6 +795,7 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked {
 
 - (BOOL)isAutoSoftwareFallbackActiveLocked {
     return _encoderMode == XCWVideoEncoderModeAuto &&
+        _autoSoftwareFallbackUntilUs != 0 &&
         _activeEncoderMode == XCWVideoEncoderModeH264Software;
 }
 
@@ -793,38 +808,63 @@ - (void)resetAutoFallbackLatencyStateLocked {
     _wasOverloaded = NO;
 }
 
+- (void)switchActiveEncoderModeLocked:(XCWVideoEncoderMode)mode {
+    if (_activeEncoderMode == mode) {
+        return;
+    }
+    _activeEncoderMode = mode;
+    _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode);
+    if (_activeEncoderMode == XCWVideoEncoderModeH264Software) {
+        _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
+        _softwareHealthyFrameCount = 0;
+        _softwarePacedFrameCount = 0;
+    } else {
+        _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
+        _hardwarePacedFrameCount = 0;
+    }
+    [self invalidateCompressionSessionLocked];
+    [self resetAutoFallbackLatencyStateLocked];
+    _needsKeyFrame = YES;
+}
+
+- (void)updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)nowUs {
+    if (!_clientForeground) {
+        [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software];
+        return;
+    }
+    if (_encoderMode == XCWVideoEncoderModeAuto &&
+        _autoSoftwareFallbackUntilUs != 0 &&
+        nowUs < _autoSoftwareFallbackUntilUs) {
+        [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software];
+        return;
+    }
+    if (_encoderMode == XCWVideoEncoderModeAuto && _autoSoftwareFallbackUntilUs != 0) {
+        _autoSoftwareFallbackUntilUs = 0;
+        _autoHardwareRetryCount += 1;
+    }
+    [self switchActiveEncoderModeLocked:_encoderMode];
+}
+
 - (void)enterAutoSoftwareFallbackLockedAtTimeUs:(uint64_t)nowUs {
     if (_encoderMode != XCWVideoEncoderModeAuto ||
         _activeEncoderMode == XCWVideoEncoderModeH264Software) {
         return;
     }
-    _activeEncoderMode = XCWVideoEncoderModeH264Software;
-    _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode);
     _autoSoftwareFallbackUntilUs = nowUs + XCWAutoHardwareRetryIntervalUs;
     _autoSoftwareFallbackCount += 1;
-    _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
-    _softwareHealthyFrameCount = 0;
-    _softwarePacedFrameCount = 0;
-    [self invalidateCompressionSessionLocked];
-    [self resetAutoFallbackLatencyStateLocked];
-    _needsKeyFrame = YES;
+    [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software];
 }
 
 - (void)retryAutoHardwareIfNeededLockedAtTimeUs:(uint64_t)nowUs {
     if (![self isAutoSoftwareFallbackActiveLocked] ||
+        !_clientForeground ||
         _autoSoftwareFallbackUntilUs == 0 ||
         nowUs < _autoSoftwareFallbackUntilUs) {
         return;
     }
-    _activeEncoderMode = XCWVideoEncoderModeAuto;
-    _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode);
     _autoSoftwareFallbackUntilUs = 0;
     _autoHardwareRetryCount += 1;
-    _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
-    _hardwarePacedFrameCount = 0;
-    [self invalidateCompressionSessionLocked];
-    [self resetAutoFallbackLatencyStateLocked];
-    _needsKeyFrame = YES;
+    [self switchActiveEncoderModeLocked:XCWVideoEncoderModeAuto];
 }
 
 - (uint64_t)activeFrameIntervalUsLocked {
diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h
@@ -31,6 +31,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData,
 - (void)requestKeyFrameRefresh;
 - (void)requestFrameRefresh;
 - (void)reconfigureVideoEncoder;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)videoEncoderStats;
 - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler;
 - (void)removeEncodedFrameListener:(id)token;
diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m
@@ -177,6 +177,14 @@ - (void)reconfigureVideoEncoder {
     [self refreshCurrentFrame];
 }
 
+- (void)setClientForeground:(BOOL)foreground {
+    [_videoEncoder setClientForeground:foreground];
+    if (foreground) {
+        [_videoEncoder requestKeyFrame];
+        [self refreshCurrentFrame];
+    }
+}
+
 - (NSDictionary *)videoEncoderStats {
     return [_videoEncoder statsRepresentation];
 }
diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h
@@ -76,6 +76,7 @@ bool xcw_native_session_start(void * _Nonnull handle, char * _Nullable * _Nullab
 void xcw_native_session_request_refresh(void * _Nonnull handle);
 void xcw_native_session_request_keyframe(void * _Nonnull handle);
 void xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle);
+void xcw_native_session_set_client_foreground(void * _Nonnull handle, bool foreground);
 char * _Nullable xcw_native_session_video_encoder_stats(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 int32_t xcw_native_session_rotation_quarter_turns(void * _Nonnull handle);
 bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);
diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m
@@ -937,6 +937,12 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) {
     }
 }
 
+void xcw_native_session_set_client_foreground(void *handle, bool foreground) {
+    @autoreleasepool {
+        [XCWNativeSessionFromHandle(handle) setClientForeground:foreground];
+    }
+}
+
 char *xcw_native_session_video_encoder_stats(void *handle, char **error_message) {
     @autoreleasepool {
         return XCWJSONStringFromObject([XCWNativeSessionFromHandle(handle) videoEncoderStats] ?: @{}, error_message);
diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h
@@ -15,6 +15,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)requestRefresh;
 - (void)requestKeyFrame;
 - (void)reconfigureVideoEncoder;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)videoEncoderStats;
 - (NSInteger)rotationQuarterTurns;
 - (BOOL)sendTouchAtX:(double)x
diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m
@@ -102,6 +102,10 @@ - (void)reconfigureVideoEncoder {
     [self.session reconfigureVideoEncoder];
 }
 
+- (void)setClientForeground:(BOOL)foreground {
+    [self.session setClientForeground:foreground];
+}
+
 - (NSDictionary *)videoEncoderStats {
     return [self.session videoEncoderStats];
 }
diff --git a/client/src/api/types.ts b/client/src/api/types.ts
@@ -1,6 +1,12 @@
 export interface EncoderStats {
+  activeEncoderMode?: string;
   averageEncodeLatencyUs?: number;
   averageEncoderLoadPercent?: number;
+  autoHardwareRetries?: number;
+  autoSoftwareFallbackActive?: boolean;
+  autoSoftwareFallbackRemainingUs?: number;
+  autoSoftwareFallbacks?: number;
+  clientForeground?: boolean;
   consecutiveOverBudgetFrames?: number;
   encoderBudgetUs?: number;
   encoderLoadPercent?: number;
diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts
@@ -86,7 +86,9 @@ export function sendStreamClientStats(stats: unknown): boolean {
 }
 
 export function sendWebRtcStreamControl(options: {
+  clientId?: string;
   forceKeyframe?: boolean;
+  foreground?: boolean;
   snapshot?: boolean;
 }): boolean {
   return sendDataChannelMessage(
@@ -3018,7 +3020,12 @@ export class StreamWorkerClient {
     return (await this.backend?.collectVisualArtifactSample?.(udid)) ?? null;
   }
 
-  sendStreamControl(options: { forceKeyframe?: boolean; snapshot?: boolean }) {
+  sendStreamControl(options: {
+    clientId?: string;
+    forceKeyframe?: boolean;
+    foreground?: boolean;
+    snapshot?: boolean;
+  }) {
     return Boolean(
       this.backend?.sendControl?.({ ...options, type: "streamControl" }),
     );
diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts
@@ -99,6 +99,10 @@ function currentClientBundle(): string {
   );
 }
 
+function isDocumentForeground(): boolean {
+  return document.visibilityState === "visible" && document.hasFocus();
+}
+
 export function useLiveStream({
   canvasElement,
   paused = false,
@@ -346,6 +350,40 @@ export function useLiveStream({
     streamTransport,
   ]);
 
+  useEffect(() => {
+    if (!simulator?.udid || paused) {
+      return;
+    }
+
+    const sendForegroundState = (foreground = isDocumentForeground()) => {
+      workerClientRef.current?.sendStreamControl({
+        clientId: clientTelemetryIdRef.current,
+        foreground,
+      });
+    };
+
+    const sendCurrentForegroundState = () => {
+      sendForegroundState();
+    };
+    const sendBackgroundState = () => {
+      sendForegroundState(false);
+    };
+
+    sendCurrentForegroundState();
+    document.addEventListener("visibilitychange", sendCurrentForegroundState);
+    window.addEventListener("focus", sendCurrentForegroundState);
+    window.addEventListener("blur", sendCurrentForegroundState);
+    window.addEventListener("pageshow", sendCurrentForegroundState);
+    window.addEventListener("pagehide", sendBackgroundState);
+    return () => {
+      document.removeEventListener("visibilitychange", sendCurrentForegroundState);
+      window.removeEventListener("focus", sendCurrentForegroundState);
+      window.removeEventListener("blur", sendCurrentForegroundState);
+      window.removeEventListener("pageshow", sendCurrentForegroundState);
+      window.removeEventListener("pagehide", sendBackgroundState);
+    };
+  }, [paused, simulator?.udid, streamTransport]);
+
   useEffect(() => {
     if (
       streamConfigApplyKey <= 0 ||
@@ -410,6 +448,10 @@ export function useLiveStream({
         visualSampleCount: latestVisualArtifactSampleCountRef.current,
         visibilityState: document.visibilityState,
       };
+      workerClientRef.current?.sendStreamControl({
+        clientId: clientTelemetryIdRef.current,
+        foreground: isDocumentForeground(),
+      });
       if (
         sendStreamClientStats(payload) ||
         remote ||
diff --git a/client/src/features/toolbar/DebugPanel.tsx b/client/src/features/toolbar/DebugPanel.tsx
@@ -80,6 +80,16 @@ export function DebugPanel({
   if (encoder) {
     rows.push(
       { label: "Encoder", value: encoder.encoderMode ?? "—" },
+      { label: "Active Encoder", value: encoder.activeEncoderMode ?? "—" },
+      {
+        label: "Client Foreground",
+        value:
+          typeof encoder.clientForeground === "boolean"
+            ? encoder.clientForeground
+              ? "yes"
+              : "no"
+            : "—",
+      },
       { label: "Encoder State", value: encoder.overloadState ?? "—" },
       {
         label: "Encoder Load",
diff --git a/docs/api/health.md b/docs/api/health.md
@@ -58,6 +58,7 @@ Returns a snapshot of every server-side counter and the rolling buffer of client
       "encoder": {
         "encoderMode": "auto",
         "activeEncoderMode": "hardware",
+        "clientForeground": true,
         "autoSoftwareFallbackActive": false,
         "hardwareAccelerated": true,
         "overloadState": "nominal",
@@ -119,7 +120,10 @@ from hardware to software encoding. When `encoderMode` is `auto`, SimDeck uses
 this signal to temporarily rebuild the active session with software H.264 if the
 hardware encoder is overloaded, then retries hardware after a cooldown. The
 `activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and
-`autoHardwareRetries` fields expose that state.
+`autoHardwareRetries` fields expose that state. `clientForeground` is `false`
+when all current browser viewers for the simulator are hidden or unfocused; in
+that state the native session uses software H.264 until a viewer returns
+foreground.
 
 ### Client stream stats
 
diff --git a/docs/api/rest.md b/docs/api/rest.md
@@ -220,6 +220,14 @@ stream attached to that peer:
 { "type": "streamControl", "forceKeyframe": true }
 ```
 
+Clients can also report page focus/visibility through stream control. When all
+current viewers for a simulator report `foreground: false`, the native session
+uses software H.264 until a viewer returns foreground:
+
+```json
+{ "type": "streamControl", "clientId": "browser", "foreground": false }
+```
+
 ```json
 { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } }
 ```
@@ -258,6 +266,10 @@ socket:
 { "type": "streamControl", "forceKeyframe": true }
 ```
 
+```json
+{ "type": "streamControl", "clientId": "browser", "foreground": false }
+```
+
 ```json
 { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } }
 ```
diff --git a/docs/guide/video.md b/docs/guide/video.md
@@ -36,6 +36,9 @@ In `auto`, active simulator encoder metrics also report `activeEncoderMode`.
 When the hardware encoder is overloaded, `auto` rebuilds the active compression
 session as software H.264, then periodically retries hardware and stays there
 when the measured encode latency returns to budget.
+If all current browser viewers for a simulator are backgrounded or unfocused,
+the active session also switches to software H.264 until at least one viewer is
+foreground again.
 The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality.
 
 ## Remote WebRTC ICE
@@ -124,6 +127,7 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques
 A few practical guidelines:
 
 - **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Auto mode moves active sessions to software when hardware H.264 is overloaded, then retries hardware after a cooldown.
+- **Backgrounded web clients use software.** The browser sends page visibility/focus over the stream control channel. If every active viewer for a simulator is hidden or unfocused, the native session uses software H.264 until a viewer returns foreground.
 - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames.
 - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency.
 - **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder.
@@ -168,7 +172,9 @@ means the shared VideoToolbox encoder is saturated; lower resolution/FPS or
 switch to software H.264. In `auto`, `activeEncoderMode`,
 `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and
 `autoHardwareRetries` show whether the active session is temporarily using
-software H.264 while the hardware path cools down.
+software H.264 while the hardware path cools down. `clientForeground` shows
+whether any current browser viewer is visible and focused; when it is `false`,
+the active encoder is software regardless of the requested mode.
 
 Clients can also push their decoder/renderer stats back to the server. Browser
 clients normally send these over the WebRTC telemetry data channel or the H264
diff --git a/server/native_stubs.c b/server/native_stubs.c
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
diff --git a/server/src/main.rs b/server/src/main.rs
diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs
diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs
diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs
diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs

Original file line number	Diff line number	Diff line change
`@@ -177,6 +177,14 @@ - (void)reconfigureVideoEncoder {`
`177`	`177`	`[self refreshCurrentFrame];`
`178`	`178`	`}`
`179`	`179`
	`180`	`+- (void)setClientForeground:(BOOL)foreground {`
	`181`	`+ [_videoEncoder setClientForeground:foreground];`
	`182`	`+ if (foreground) {`
	`183`	`+ [_videoEncoder requestKeyFrame];`
	`184`	`+ [self refreshCurrentFrame];`
	`185`	`+ }`
	`186`	`+}`
	`187`	`+`
`180`	`188`	`- (NSDictionary *)videoEncoderStats {`
`181`	`189`	`return [_videoEncoder statsRepresentation];`
`182`	`190`	`}`
Original file line number	Diff line number	Diff line change
`@@ -937,6 +937,12 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) {`
`937`	`937`	`}`
`938`	`938`	`}`
`939`	`939`
	`940`	`+void xcw_native_session_set_client_foreground(void *handle, bool foreground) {`
	`941`	`+ @autoreleasepool {`
	`942`	`+ [XCWNativeSessionFromHandle(handle) setClientForeground:foreground];`
	`943`	`+ }`
	`944`	`+}`
	`945`	`+`
`940`	`946`	`char xcw_native_session_video_encoder_stats(void handle, char **error_message) {`
`941`	`947`	`@autoreleasepool {`
`942`	`948`	`return XCWJSONStringFromObject([XCWNativeSessionFromHandle(handle) videoEncoderStats] ?: @{}, error_message);`
Original file line number	Diff line number	Diff line change
`@@ -102,6 +102,10 @@ - (void)reconfigureVideoEncoder {`
`102`	`102`	`[self.session reconfigureVideoEncoder];`
`103`	`103`	`}`
`104`	`104`
	`105`	`+- (void)setClientForeground:(BOOL)foreground {`
	`106`	`+ [self.session setClientForeground:foreground];`
	`107`	`+}`
	`108`	`+`
`105`	`109`	`- (NSDictionary *)videoEncoderStats {`
`106`	`110`	`return [self.session videoEncoderStats];`
`107`	`111`	`}`