Add stream encoder and quality controls to the UI

Author: DjDeveloperr

README.md

@@ -81,7 +81,7 @@ the outbound bridge alive until you press Ctrl-C. It uses software H.264 by
 default with realtime stream settings for remote viewing, and prints the active
 codec/profile when it starts. Studio defaults to the `smooth` stream quality
 profile (`1170` longest edge, dynamic up to `60` fps). Use
-`--stream-quality quality|balanced|smooth|economy|ci-software` to override it,
+`--stream-quality quality|balanced|fast|smooth|economy|ci-software` to override it,
 or pass `--video-codec hardware` when a dedicated hardware encoder is preferable.
 The remote viewer renders live video with the browser's native video element;
 the canvas is only used for input geometry.
@@ -116,11 +116,12 @@ more important than full-resolution smoothness:
 simdeck daemon start --video-codec software --low-latency
 ```
 
-Local browser streams default to 60 fps. On high-refresh local displays, opt in
-to a paced hardware stream up to 120 fps:
+Local browser streams default to realtime WebRTC delivery with the `quality`
+profile on VideoToolbox H.264: full resolution, 120 fps, and a high bitrate floor. On
+high-refresh local displays, raise the local stream target explicitly:
 
 ```sh
-simdeck daemon restart --local-stream-fps 120
+simdeck daemon restart --local-stream-fps 240
 ```
 
 Restart the CoreSimulator service layer when `simctl` reports a stale service

cli/DFPrivateSimulatorDisplayBridge.m

@@ -3372,13 +3372,15 @@ - (void)disconnect {
             self->_latestPixelBuffer = nil;
         }
 
-        [self->_headlessHostWindow orderOut:nil];
-        [self->_headlessHostWindow close];
-        self->_headlessHostWindow = nil;
-        self->_headlessHostView = nil;
+        DFRunOnMainSync(^{
+            [self->_headlessHostWindow orderOut:nil];
+            [self->_headlessHostWindow close];
+            self->_headlessHostWindow = nil;
+            self->_headlessHostView = nil;
+            [self.displayView removeFromSuperview];
+        });
 
         [self updateStatus:@"Disconnected"];
-        [self.displayView removeFromSuperview];
     };
 
     if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) {

cli/XCWH264Encoder.m

@@ -163,8 +163,8 @@ - (void)refreshCurrentFrame {
 }
 
 - (void)requestKeyFrameRefresh {
-    [self refreshCurrentFrame];
     [_videoEncoder requestKeyFrame];
+    [self refreshCurrentFrame];
 }
 
 - (void)requestFrameRefresh {
@@ -189,18 +189,9 @@ - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler {
     NSUUID *token = [NSUUID UUID];
     dispatch_sync(_stateQueue, ^{
         self->_encodedFrameListeners[token] = [handler copy];
-        if (self->_latestKeyFrameData.length > 0) {
-            handler(self->_latestKeyFrameData,
-                    self->_latestKeyFrameSequenceValue,
-                    self->_latestKeyFrameTimestampUs,
-                    YES,
-                    self->_latestKeyFrameCodec,
-                    self->_latestKeyFrameDecoderConfig,
-                    self->_latestKeyFrameDimensions);
-        }
     });
-    [self refreshCurrentFrame];
     [_videoEncoder requestKeyFrame];
+    [self refreshCurrentFrame];
     return token;
 }
 
@@ -209,7 +200,7 @@ - (void)removeEncodedFrameListener:(id)token {
         return;
     }
 
-    dispatch_async(_stateQueue, ^{
+    dispatch_sync(_stateQueue, ^{
         [self->_encodedFrameListeners removeObjectForKey:(NSUUID *)token];
     });
 }

cli/XCWPrivateSimulatorSession.m

@@ -40,6 +40,12 @@ import { usePointerInput } from "../features/input/usePointerInput";
 import { simulatorRuntimeLabel } from "../features/simulators/simulatorDisplay";
 import { useSimulatorList } from "../features/simulators/useSimulatorList";
 import { sendWebRtcControlMessage } from "../features/stream/streamWorkerClient";
+import type {
+  StreamConfig,
+  StreamEncoder,
+  StreamFps,
+  StreamQualityPreset,
+} from "../features/stream/streamTypes";
 import { useLiveStream } from "../features/stream/useLiveStream";
 import { DebugPanel } from "../features/toolbar/DebugPanel";
 import { Toolbar } from "../features/toolbar/Toolbar";
@@ -86,6 +92,16 @@ const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500;
 const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10;
 const LOGICAL_INSPECTOR_MAX_DEPTH = 80;
 const AUTH_REQUIRED_MESSAGE = "SimDeck API access token is required.";
+const LOCAL_STREAM_DEFAULTS: StreamConfig = {
+  encoder: "hardware",
+  fps: 120,
+  quality: "quality",
+};
+const REMOTE_STREAM_DEFAULTS: StreamConfig = {
+  encoder: "software",
+  fps: 30,
+  quality: "balanced",
+};
 clearLegacyVolatileUiState();
 
 function buildChromeUrl(udid: string, stamp: number): string {
@@ -277,6 +293,9 @@ export function AppShell({
   const [touchOverlayVisible, setTouchOverlayVisible] = useState(() =>
     readStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, true),
   );
+  const [streamConfig, setStreamConfig] = useState<StreamConfig>(() =>
+    remoteStream ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS,
+  );
   const [touchIndicators, setTouchIndicators] = useState<TouchIndicator[]>([]);
 
   const menuRef = useRef<HTMLDivElement | null>(null);
@@ -390,8 +409,21 @@ export function AppShell({
     canvasElement: streamCanvasElement,
     remote: remoteStream,
     simulator: selectedSimulator,
+    streamConfig,
   });
 
+  const updateStreamEncoder = useCallback((encoder: StreamEncoder) => {
+    setStreamConfig((current) => ({ ...current, encoder }));
+  }, []);
+
+  const updateStreamFps = useCallback((fps: StreamFps) => {
+    setStreamConfig((current) => ({ ...current, fps }));
+  }, []);
+
+  const updateStreamQuality = useCallback((quality: StreamQualityPreset) => {
+    setStreamConfig((current) => ({ ...current, quality }));
+  }, []);
+
   useEffect(() => {
     if (
       !selectedSimulator ||
@@ -880,9 +912,7 @@ export function AppShell({
     ? streamStatus.detail
       ? `${streamStatus.error} ${streamStatus.detail}`
       : streamStatus.error
-    : streamStatus.state === "connecting" && !hasFrame
-      ? (streamStatus.detail ?? "")
-      : "";
+    : "";
   const viewportStatusOverlayLabel =
     simulatorStatusOverlayLabel ||
     streamStatusMessage ||
@@ -1392,6 +1422,9 @@ export function AppShell({
             setStreamStamp(Date.now());
           }, false);
         }}
+        onStreamEncoderChange={updateStreamEncoder}
+        onStreamFpsChange={updateStreamFps}
+        onStreamQualityChange={updateStreamQuality}
         onShutdown={() => {
           if (!selectedSimulator) {
             return;
@@ -1438,6 +1471,7 @@ export function AppShell({
           !selectedSimulator.isBooted &&
           !selectedSimulatorTransitionKind,
         )}
+        streamConfig={streamConfig}
         showStopButton={Boolean(
           selectedSimulator?.isBooted && !selectedSimulatorTransitionKind,
         )}

client/src/app/AppShell.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react";
+import { useRef, useState } from "react";
 
 import type { ChromeProfile, TouchPhase } from "../../api/types";
 import { normalizedPointerCoordinatesForOrientation } from "./gestureMath";
@@ -33,48 +33,14 @@ export function usePointerInput({
   onTouchPreview,
 }: UsePointerInputOptions) {
   const activePointerRef = useRef<number | null>(null);
-  const moveFrameRef = useRef<number>(0);
   const panningRef = useRef<{
     startX: number;
     startY: number;
     startPanX: number;
     startPanY: number;
   } | null>(null);
-  const queuedMoveRef = useRef<Point | null>(null);
   const [isPanning, setIsPanning] = useState(false);
 
-  useEffect(() => {
-    return () => {
-      if (moveFrameRef.current) {
-        cancelAnimationFrame(moveFrameRef.current);
-      }
-    };
-  }, []);
-
-  function queueMove(coords: Point) {
-    queuedMoveRef.current = coords;
-    if (moveFrameRef.current) {
-      return;
-    }
-
-    moveFrameRef.current = requestAnimationFrame(() => {
-      moveFrameRef.current = 0;
-      const nextCoords = queuedMoveRef.current;
-      queuedMoveRef.current = null;
-      if (nextCoords) {
-        onTouch("moved", nextCoords);
-      }
-    });
-  }
-
-  function clearQueuedMove() {
-    if (moveFrameRef.current) {
-      cancelAnimationFrame(moveFrameRef.current);
-      moveFrameRef.current = 0;
-    }
-    queuedMoveRef.current = null;
-  }
-
   function startPanning(event: React.PointerEvent<HTMLElement>) {
     if (event.pointerType !== "mouse") {
       return;
@@ -155,7 +121,7 @@ export function usePointerInput({
     );
     if (coords) {
       onTouchPreview?.("moved", coords);
-      queueMove(coords);
+      onTouch("moved", coords);
     }
   }
 
@@ -168,7 +134,6 @@ export function usePointerInput({
       return;
     }
     activePointerRef.current = null;
-    clearQueuedMove();
     const coords = normalizedPointerCoordinatesForOrientation(
       event,
       rotationQuarterTurns,

client/src/features/input/usePointerInput.ts

@@ -1,6 +1,12 @@
 import type { RefObject } from "react";
 
 import type { SimulatorMetadata } from "../../api/types";
+import type {
+  StreamConfig,
+  StreamEncoder,
+  StreamFps,
+  StreamQualityPreset,
+} from "../stream/streamTypes";
 import { SimulatorRow } from "./SimulatorRow";
 
 interface SimulatorMenuProps {
@@ -16,13 +22,17 @@ interface SimulatorMenuProps {
   onOpenBundlePrompt: () => void;
   onOpenUrlPrompt: () => void;
   onRotateLeft: () => void;
+  onStreamEncoderChange: (encoder: StreamEncoder) => void;
+  onStreamFpsChange: (fps: StreamFps) => void;
+  onStreamQualityChange: (quality: StreamQualityPreset) => void;
   onToggleAppearance: () => void;
   onToggleDebug: () => void;
   onToggleMenu: () => void;
   onToggleTouchOverlay: () => void;
   search: string;
   selectedSimulator: SimulatorMetadata | null;
   setSelectedUDID: (udid: string) => void;
+  streamConfig: StreamConfig;
   touchOverlayVisible: boolean;
 }
 
@@ -39,13 +49,17 @@ export function SimulatorMenu({
   onOpenBundlePrompt,
   onOpenUrlPrompt,
   onRotateLeft,
+  onStreamEncoderChange,
+  onStreamFpsChange,
+  onStreamQualityChange,
   onToggleAppearance,
   onToggleDebug,
   onToggleMenu,
   onToggleTouchOverlay,
   search,
   selectedSimulator,
   setSelectedUDID,
+  streamConfig,
   touchOverlayVisible,
 }: SimulatorMenuProps) {
   return (
@@ -94,6 +108,46 @@ export function SimulatorMenu({
           ) : null}
           {selectedSimulator ? (
             <>
+              <div className="menu-divider" />
+              <div className="menu-section">
+                <span className="menu-section-title">Stream</span>
+                <div aria-label="Encoder" className="menu-segment two">
+                  {STREAM_ENCODERS.map((option) => (
+                    <button
+                      className={`menu-option ${streamConfig.encoder === option.value ? "active" : ""}`}
+                      key={option.value}
+                      onClick={() => onStreamEncoderChange(option.value)}
+                      type="button"
+                    >
+                      {option.label}
+                    </button>
+                  ))}
+                </div>
+                <div aria-label="Frame rate" className="menu-segment">
+                  {STREAM_FPS_OPTIONS.map((option) => (
+                    <button
+                      className={`menu-option ${streamConfig.fps === option.value ? "active" : ""}`}
+                      key={option.value}
+                      onClick={() => onStreamFpsChange(option.value)}
+                      type="button"
+                    >
+                      {option.label}
+                    </button>
+                  ))}
+                </div>
+                <div aria-label="Quality" className="menu-segment">
+                  {STREAM_QUALITY_OPTIONS.map((option) => (
+                    <button
+                      className={`menu-option ${streamConfig.quality === option.value ? "active" : ""}`}
+                      key={option.value}
+                      onClick={() => onStreamQualityChange(option.value)}
+                      type="button"
+                    >
+                      {option.label}
+                    </button>
+                  ))}
+                </div>
+              </div>
               <div className="menu-divider" />
               <div className="menu-actions">
                 <button className="menu-action" onClick={onOpenUrlPrompt}>
@@ -148,6 +202,26 @@ export function SimulatorMenu({
   );
 }
 
+const STREAM_ENCODERS: Array<{ label: string; value: StreamEncoder }> = [
+  { label: "Hardware", value: "hardware" },
+  { label: "Software", value: "software" },
+];
+
+const STREAM_FPS_OPTIONS: Array<{ label: string; value: StreamFps }> = [
+  { label: "30", value: 30 },
+  { label: "60", value: 60 },
+  { label: "120", value: 120 },
+];
+
+const STREAM_QUALITY_OPTIONS: Array<{
+  label: string;
+  value: StreamQualityPreset;
+}> = [
+  { label: "Quality", value: "quality" },
+  { label: "Balanced", value: "balanced" },
+  { label: "Fast", value: "fast" },
+];
+
 function MenuIcon() {
   return (
     <svg fill="currentColor" height="16" viewBox="0 0 16 16" width="16">

client/src/features/simulators/SimulatorMenu.tsx

@@ -6,12 +6,15 @@ export function createEmptyStreamStats(): StreamStats {
     codec: "",
     decodeQueueSize: 0,
     decodedFrames: 0,
+    decoderDroppedFrames: 0,
     droppedFrames: 0,
     frameSequence: 0,
     height: 0,
     latestFrameGapMs: 0,
     latestRenderMs: 0,
     maxRenderMs: 0,
+    packetsLost: 0,
+    presentationDroppedFrames: 0,
     receivedPackets: 0,
     reconnects: 0,
     renderedFrames: 0,