Improve SimDeck stream readiness signals

Author: DjDeveloperr

client/src/api/simulators.ts

@@ -7,6 +7,7 @@ import type {
   InspectorRequestResponse,
   SimulatorLogsResponse,
   SimulatorMetadata,
+  SimulatorStateResponse,
   SimulatorsResponse,
   WebKitTargetDiscovery,
 } from "./types";
@@ -22,6 +23,16 @@ export async function fetchChromeProfile(udid: string): Promise<ChromeProfile> {
   return apiRequest<ChromeProfile>(`/api/simulators/${udid}/chrome-profile`);
 }
 
+export async function fetchSimulatorState(
+  udid: string,
+  options: RequestInit = {},
+): Promise<SimulatorStateResponse> {
+  return apiRequest<SimulatorStateResponse>(
+    `/api/simulators/${encodeURIComponent(udid)}/state`,
+    options,
+  );
+}
+
 export async function fetchAccessibilityTree(
   udid: string,
   source: AccessibilitySourcePreference = "auto",

client/src/api/types.ts

@@ -22,6 +22,7 @@ export interface PrivateDisplayInfo {
   displayHeight: number;
   encoder?: EncoderStats;
   frameSequence: number;
+  lastFrameAt?: number;
   rotationQuarterTurns?: number;
 }
 
@@ -110,6 +111,24 @@ export interface SimulatorResponse {
   simulator: SimulatorMetadata;
 }
 
+export interface SimulatorForegroundApp {
+  appName?: string | null;
+  bundleIdentifier?: string | null;
+  processIdentifier: number;
+}
+
+export interface SimulatorStateResponse {
+  booted: boolean;
+  displayReady: boolean;
+  displayStatus: string;
+  foregroundApp?: SimulatorForegroundApp | null;
+  frameSequence: number;
+  lastFrameAgeMs?: number | null;
+  lastFrameAt: number;
+  simulator: SimulatorMetadata;
+  udid: string;
+}
+
 export interface ChromeProfile {
   totalWidth: number;
   totalHeight: number;

client/src/app/AppShell.tsx

@@ -30,14 +30,19 @@ import {
   toggleAppearance,
   type ControlMessage,
 } from "../api/controls";
-import { fetchAccessibilityTree, fetchChromeProfile } from "../api/simulators";
+import {
+  fetchAccessibilityTree,
+  fetchChromeProfile,
+  fetchSimulatorState,
+} from "../api/simulators";
 import type {
   AccessibilityNode,
   AccessibilitySource,
   AccessibilitySourcePreference,
   AccessibilityTreeResponse,
   ChromeProfile,
   SimulatorMetadata,
+  SimulatorStateResponse,
   TouchPhase,
 } from "../api/types";
 import { AccessibilityInspector } from "../features/accessibility/AccessibilityInspector";
@@ -430,6 +435,8 @@ export function AppShell({
   const [streamConfigApplyKey, setStreamConfigApplyKey] = useState(0);
   const [streamConfigReady, setStreamConfigReady] = useState(false);
   const [touchIndicators, setTouchIndicators] = useState<TouchIndicator[]>([]);
+  const [selectedSimulatorState, setSelectedSimulatorState] =
+    useState<SimulatorStateResponse | null>(null);
 
   const menuRef = useRef<HTMLDivElement | null>(null);
   const outerCanvasRef = useRef<HTMLDivElement | null>(null);
@@ -494,7 +501,7 @@ export function AppShell({
       .includes(searchNeedle);
   });
 
-  const selectedSimulator =
+  const baseSelectedSimulator =
     (fixedSimulatorUDID
       ? (simulators.find(
           (simulator) => simulator.udid === fixedSimulatorUDID,
@@ -511,6 +518,14 @@ export function AppShell({
     filteredSimulators.find((simulator) => simulator.isBooted) ??
     filteredSimulators[0] ??
     null;
+  const selectedSimulator =
+    baseSelectedSimulator &&
+    selectedSimulatorState?.udid === baseSelectedSimulator.udid
+      ? {
+          ...baseSelectedSimulator,
+          ...selectedSimulatorState.simulator,
+        }
+      : baseSelectedSimulator;
   const selectedSimulatorTransitionKind =
     selectedSimulator != null &&
     simulatorTransition?.udid === selectedSimulator.udid
@@ -532,6 +547,59 @@ export function AppShell({
         : "Stopping..."
       : "";
 
+  useEffect(() => {
+    const udid = baseSelectedSimulator?.udid;
+    if (!udid) {
+      setSelectedSimulatorState(null);
+      return;
+    }
+
+    let cancelled = false;
+    let timeoutId = 0;
+    let controller: AbortController | null = null;
+
+    const loadState = () => {
+      controller?.abort();
+      controller =
+        typeof AbortController !== "undefined" ? new AbortController() : null;
+      void fetchSimulatorState(
+        udid,
+        controller ? { signal: controller.signal } : {},
+      )
+        .then((state) => {
+          if (!cancelled) {
+            setSelectedSimulatorState(state);
+          }
+        })
+        .catch((error) => {
+          if (
+            !cancelled &&
+            !(error instanceof DOMException && error.name === "AbortError")
+          ) {
+            setSelectedSimulatorState(null);
+          }
+        })
+        .finally(() => {
+          if (!cancelled) {
+            timeoutId = window.setTimeout(
+              loadState,
+              baseSelectedSimulator?.isBooted ? 1000 : 3000,
+            );
+          }
+        });
+    };
+
+    setSelectedSimulatorState(null);
+    loadState();
+    return () => {
+      cancelled = true;
+      if (timeoutId) {
+        window.clearTimeout(timeoutId);
+      }
+      controller?.abort();
+    };
+  }, [baseSelectedSimulator?.isBooted, baseSelectedSimulator?.udid]);
+
   const handleStreamCanvasRef = useCallback(
     (node: HTMLCanvasElement | null) => {
       streamCanvasRef.current = node;
@@ -1354,9 +1422,29 @@ export function AppShell({
       ? `${visibleStreamError} ${streamStatus.detail}`
       : visibleStreamError
     : "";
+  const serverFrameSequence =
+    selectedSimulatorState?.frameSequence ??
+    selectedSimulator?.privateDisplay?.frameSequence ??
+    0;
+  const browserFramePending = Boolean(
+    selectedSimulator?.isBooted &&
+    serverFrameSequence > 0 &&
+    !hasFrame &&
+    streamStatus.state !== "error" &&
+    !visibleStreamError,
+  );
+  const streamStatusLabel = streamAgentStatusLabel({
+    hasFrame,
+    simulator: selectedSimulator,
+    simulatorState: selectedSimulatorState,
+    stats,
+    streamStatusDetail: streamStatus.detail,
+    streamStatusState: streamStatus.state,
+  });
   const viewportStatusOverlayLabel =
     simulatorStatusOverlayLabel ||
     streamStatusMessage ||
+    (browserFramePending ? "Stream connected, browser frame pending" : "") ||
     (selectedSimulator ? visibleListError : "");
   const viewportHasStreamError = Boolean(
     streamStatus.state === "error" ||
@@ -2117,6 +2205,7 @@ export function AppShell({
         chromeScreenStyle={viewportScreenStyle}
         chromeUrl={chromeUrl}
         chromeButtonUrl={chromeButtonUrl}
+        browserFramePending={browserFramePending}
         debugPanel={
           debugVisible ? (
             <DebugPanel
@@ -2180,6 +2269,7 @@ export function AppShell({
         streamCanvasRef={handleStreamCanvasRef}
         streamBackend={streamBackend}
         streamCanvasKey={streamCanvasKey}
+        streamStatusLabel={streamStatusLabel}
         statusOverlayLabel={viewportStatusOverlayLabel}
         touchIndicators={touchIndicators}
         touchOverlayVisible={touchOverlayVisible}
@@ -2315,6 +2405,76 @@ function friendlyStreamError(
   return friendlyClientError(normalized);
 }
 
+function streamAgentStatusLabel({
+  hasFrame,
+  simulator,
+  simulatorState,
+  stats,
+  streamStatusDetail,
+  streamStatusState,
+}: {
+  hasFrame: boolean;
+  simulator: SimulatorMetadata | null;
+  simulatorState: SimulatorStateResponse | null;
+  stats: {
+    frameSequence: number;
+    latestFrameGapMs: number;
+    renderedFrames: number;
+  };
+  streamStatusDetail?: string;
+  streamStatusState: string;
+}): string {
+  if (!simulator) {
+    return "No simulator selected";
+  }
+
+  const display = simulator.privateDisplay;
+  const serverFrameSequence =
+    simulatorState?.frameSequence ?? display?.frameSequence ?? 0;
+  const serverFrameAgeMs =
+    simulatorState?.lastFrameAgeMs ??
+    (display?.lastFrameAt
+      ? Math.max(0, Date.now() - display.lastFrameAt)
+      : null);
+  const foreground =
+    simulatorState?.foregroundApp?.appName ??
+    simulatorState?.foregroundApp?.bundleIdentifier ??
+    "unknown foreground app";
+  const parts = [
+    simulator.isBooted ? "Booted" : "Shutdown",
+    display?.displayReady || simulatorState?.displayReady
+      ? "display ready"
+      : "display not ready",
+    `server ${display?.displayStatus ?? simulatorState?.displayStatus ?? "Unknown"}`,
+    `server frame ${serverFrameSequence}`,
+    serverFrameAgeMs == null
+      ? "server frame age unknown"
+      : `server frame ${formatMilliseconds(serverFrameAgeMs)} ago`,
+    hasFrame
+      ? stats.frameSequence > 0
+        ? `browser frame ${stats.frameSequence}`
+        : `browser rendered ${stats.renderedFrames}`
+      : "browser frame pending",
+    `browser gap ${formatMilliseconds(stats.latestFrameGapMs)}`,
+    `foreground ${foreground}`,
+    `client ${streamStatusState}`,
+  ];
+  if (streamStatusDetail) {
+    parts.push(streamStatusDetail);
+  }
+  return parts.join(" · ");
+}
+
+function formatMilliseconds(value: number): string {
+  if (!Number.isFinite(value) || value <= 0) {
+    return "0ms";
+  }
+  if (value < 1000) {
+    return `${Math.round(value)}ms`;
+  }
+  return `${(value / 1000).toFixed(1)}s`;
+}
+
 function userFacingAccessibilityError(message: string): string {
   const normalized = message.trim();
   if (!normalized) {

client/src/features/stream/streamWorkerClient.ts

@@ -1314,6 +1314,7 @@ class WebRtcStreamClient implements StreamClientBackend {
   private reportedVideoWidth = 0;
   private receiverStatsInterval = 0;
   private receiverStatsSeen = false;
+  private streamingReported = false;
   private shouldReconnect = false;
   private latestRgbaSequence = -1;
   private rgbaAssemblies = new Map<number, WebRtcRgbaAssembly>();
@@ -1413,6 +1414,7 @@ class WebRtcStreamClient implements StreamClientBackend {
     this.reportedVideoHeight = 0;
     this.reportedVideoWidth = 0;
     this.receiverStatsSeen = false;
+    this.streamingReported = false;
     this.onMessage({
       type: "status",
       status: { detail: "Creating WebRTC offer", state: "connecting" },
@@ -1544,17 +1546,7 @@ class WebRtcStreamClient implements StreamClientBackend {
           this.clearIceRestartTimeout();
           this.iceRestartInFlight = false;
           this.reconnectDelayMs = WEBRTC_RECONNECT_BASE_DELAY_MS;
-          if (this.reportedVideoWidth > 0 && this.reportedVideoHeight > 0) {
-            this.onMessage({
-              type: "status",
-              status: {
-                detail: useRgbaTransport
-                  ? "WebRTC RGBA stream connected"
-                  : "WebRTC media connected",
-                state: "streaming",
-              },
-            });
-          }
+          this.reportWebRtcStreaming();
           return;
         }
         if (peerConnection.connectionState === "disconnected") {
@@ -1818,6 +1810,7 @@ class WebRtcStreamClient implements StreamClientBackend {
     const reconnects = this.stats.reconnects;
     this.hasRenderedFrame = false;
     this.lastVideoFrameAt = 0;
+    this.streamingReported = false;
     this.stats = createEmptyStreamStats();
     this.stats.iceRestartReason = iceRestartReason;
     this.stats.iceRestarts = iceRestarts;
@@ -2290,6 +2283,7 @@ class WebRtcStreamClient implements StreamClientBackend {
       previousFrameAt > 0 ? finishedAt - previousFrameAt : 0;
     this.reportVideoConfig(frame.width, frame.height);
     this.onMessage({ type: "stats", stats: { ...this.stats } });
+    this.reportWebRtcStreaming();
   }
 
   private drawVideoFrame = () => {
@@ -2320,6 +2314,7 @@ class WebRtcStreamClient implements StreamClientBackend {
       }
       this.lastVideoFrameAt = now;
       this.onMessage({ type: "stats", stats: { ...this.stats } });
+      this.reportWebRtcStreaming();
     }
     this.scheduleVideoFrame();
   };
@@ -2355,12 +2350,19 @@ class WebRtcStreamClient implements StreamClientBackend {
       type: "video-config",
       size: { height, width },
     });
+  }
+
+  private reportWebRtcStreaming() {
+    if (!this.hasRenderedFrame || this.streamingReported) {
+      return;
+    }
+    this.streamingReported = true;
     this.onMessage({
       type: "status",
       status: {
         detail: this.rgbaMode
-          ? "WebRTC RGBA stream connected"
-          : "WebRTC media connected",
+          ? "WebRTC RGBA first frame rendered"
+          : "WebRTC first video frame rendered",
         state: "streaming",
       },
     });

client/src/features/toolbar/Toolbar.tsx

@@ -4,6 +4,7 @@ import {
   Half2Icon as AppearanceIcon,
   HomeIcon,
   LayersIcon as HierarchyIcon,
+  Link2Icon as OpenUrlIcon,
   PlayIcon,
   ReloadIcon as RotateRightIcon,
   StopIcon,
@@ -208,6 +209,14 @@ export function Toolbar({
                 <StopIcon />
               </button>
             ) : null}
+            <button
+              aria-label="Open URL"
+              className="tbtn icon-btn"
+              onClick={onOpenUrlPrompt}
+              title="Open URL"
+            >
+              <OpenUrlIcon />
+            </button>
             <button
               aria-label="Home"
               className="tbtn icon-btn"