Route remote realtime controls over WebRTC

DjDeveloperr · DjDeveloperr · commit 350ea4497589 · 2026-05-02T00:18:27.000-04:00
diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts
@@ -11,7 +11,13 @@ import type {
 
 export type ControlMessage =
   | ({ type: "touch" } & TouchPayload)
-  | ({ type: "key" } & KeyPayload);
+  | ({ type: "key" } & KeyPayload)
+  | { type: "dismissKeyboard" }
+  | { type: "home" }
+  | { type: "appSwitcher" }
+  | { type: "rotateLeft" }
+  | { type: "rotateRight" }
+  | { type: "toggleAppearance" };
 
 async function postSimulatorAction(
   udid: string,
diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx
@@ -995,18 +995,19 @@ export function AppShell({
     return state;
   }, []);
 
-  function sendControl(udid: string, message: ControlMessage) {
+  function sendControl(udid: string, message: ControlMessage): boolean {
     setLocalError("");
     const encoded = JSON.stringify(message);
     if (sendWebRtcControlMessage(encoded)) {
-      return;
+      return true;
     }
     const state = ensureControlSocket(udid);
     if (state.socket.readyState === WebSocket.OPEN) {
       state.socket.send(encoded);
     } else {
       state.pending.push(encoded);
     }
+    return true;
   }
 
   useEffect(() => closeControlSocket, [closeControlSocket]);
@@ -1297,28 +1298,47 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
-          void runAction(() => dismissKeyboard(selectedSimulator.udid), false);
+          if (
+            !sendControl(selectedSimulator.udid, { type: "dismissKeyboard" })
+          ) {
+            void runAction(
+              () => dismissKeyboard(selectedSimulator.udid),
+              false,
+            );
+          }
         }}
         onHome={() => {
           if (!selectedSimulator) {
             return;
           }
           setAccessibilitySelectedId("");
           setAccessibilityHoveredId(null);
-          void runAction(() => pressHome(selectedSimulator.udid), false);
+          if (!sendControl(selectedSimulator.udid, { type: "home" })) {
+            void runAction(() => pressHome(selectedSimulator.udid), false);
+          }
         }}
         onOpenAppSwitcher={() => {
           if (!selectedSimulator) {
             return;
           }
           setAccessibilitySelectedId("");
           setAccessibilityHoveredId(null);
-          void runAction(() => openAppSwitcher(selectedSimulator.udid), false);
+          if (!sendControl(selectedSimulator.udid, { type: "appSwitcher" })) {
+            void runAction(
+              () => openAppSwitcher(selectedSimulator.udid),
+              false,
+            );
+          }
         }}
         onRotateLeft={() => {
           if (!selectedSimulator) {
             return;
           }
+          if (sendControl(selectedSimulator.udid, { type: "rotateLeft" })) {
+            setRotationQuarterTurns((current) => (current + 3) % 4);
+            setStreamStamp(Date.now());
+            return;
+          }
           void runAction(async () => {
             await rotateLeft(selectedSimulator.udid);
             setRotationQuarterTurns((current) => (current + 3) % 4);
@@ -1331,6 +1351,11 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
+          if (sendControl(selectedSimulator.udid, { type: "rotateRight" })) {
+            setRotationQuarterTurns((current) => (current + 1) % 4);
+            setStreamStamp(Date.now());
+            return;
+          }
           void runAction(async () => {
             await rotateRight(selectedSimulator.udid);
             setRotationQuarterTurns((current) => (current + 1) % 4);
@@ -1355,7 +1380,10 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
-          void runAction(() => toggleAppearance(selectedSimulator.udid));
+          const encoded = JSON.stringify({ type: "toggleAppearance" });
+          if (!sendWebRtcControlMessage(encoded)) {
+            void runAction(() => toggleAppearance(selectedSimulator.udid));
+          }
         }}
         onToggleDebug={() => setDebugVisible((current) => !current)}
         onToggleHierarchy={() => {
diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts
@@ -27,6 +27,16 @@ export function sendWebRtcControlMessage(encoded: string): boolean {
   return true;
 }
 
+export function sendWebRtcClientStats(stats: unknown): boolean {
+  if (activeWebRtcControlChannel?.readyState !== "open") {
+    return false;
+  }
+  activeWebRtcControlChannel.send(
+    JSON.stringify({ stats, type: "clientStats" }),
+  );
+  return true;
+}
+
 export function buildStreamTarget(
   udid: string,
   options: { remote?: boolean } = {},
@@ -532,6 +542,9 @@ class WebRtcStreamClient implements StreamClientBackend {
       url: window.location.href,
       userAgent: window.navigator.userAgent,
     };
+    if (sendWebRtcClientStats(payload) || this.remoteMode) {
+      return;
+    }
     void fetch(
       new URL(apiUrl("/api/client-stream-stats"), window.location.href),
       {
diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts
@@ -8,6 +8,7 @@ import { createEmptyStreamStats } from "./stats";
 import {
   buildStreamTarget,
   canUseWebRtc,
+  sendWebRtcClientStats,
   StreamWorkerClient,
   type StreamBackend,
 } from "./streamWorkerClient";
@@ -254,21 +255,25 @@ export function useLiveStream({
     const postTelemetry = () => {
       const latestStats = latestStatsRef.current;
       const latestStatus = latestStatusRef.current;
+      const payload = {
+        ...latestStats,
+        appFps: latestFpsRef.current,
+        clientId: clientTelemetryIdRef.current,
+        focused: document.hasFocus(),
+        kind: "page",
+        pageFps: pageFpsRef.current,
+        status: latestStatus.state,
+        timestampMs: Date.now(),
+        udid: simulator.udid,
+        url: window.location.href,
+        userAgent: window.navigator.userAgent,
+        visibilityState: document.visibilityState,
+      };
+      if (sendWebRtcClientStats(payload) || remote) {
+        return;
+      }
       void fetch(buildClientTelemetryUrl(), {
-        body: JSON.stringify({
-          ...latestStats,
-          appFps: latestFpsRef.current,
-          clientId: clientTelemetryIdRef.current,
-          focused: document.hasFocus(),
-          kind: "page",
-          pageFps: pageFpsRef.current,
-          status: latestStatus.state,
-          timestampMs: Date.now(),
-          udid: simulator.udid,
-          url: window.location.href,
-          userAgent: window.navigator.userAgent,
-          visibilityState: document.visibilityState,
-        }),
+        body: JSON.stringify(payload),
         cache: "no-store",
         headers: apiHeaders(),
         method: "POST",
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
@@ -191,6 +191,12 @@ pub(crate) enum ControlMessage {
         key_code: u16,
         modifiers: Option<u32>,
     },
+    DismissKeyboard,
+    Home,
+    AppSwitcher,
+    RotateLeft,
+    RotateRight,
+    ToggleAppearance,
 }
 
 #[derive(Deserialize)]
@@ -1279,6 +1285,14 @@ pub(crate) async fn run_control_message(
             key_code,
             modifiers,
         } => session.send_key(key_code, modifiers.unwrap_or(0)),
+        ControlMessage::DismissKeyboard => session.send_key(41, 0),
+        ControlMessage::Home => session.press_home(),
+        ControlMessage::AppSwitcher => session.open_app_switcher(),
+        ControlMessage::RotateLeft => session.rotate_left(),
+        ControlMessage::RotateRight => session.rotate_right(),
+        ControlMessage::ToggleAppearance => Err(AppError::bad_request(
+            "`toggleAppearance` requires the WebRTC control channel.",
+        )),
     })
     .await
     .map_err(|error| AppError::internal(format!("Failed to join control task: {error}")))?
diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs
@@ -686,6 +686,46 @@ impl NativeSession {
             )
         }
     }
+
+    pub fn press_home(&self) -> Result<(), AppError> {
+        unsafe {
+            let mut error = ptr::null_mut();
+            bool_result(
+                ffi::xcw_native_session_press_home(self.handle, &mut error),
+                error,
+            )
+        }
+    }
+
+    pub fn open_app_switcher(&self) -> Result<(), AppError> {
+        unsafe {
+            let mut error = ptr::null_mut();
+            bool_result(
+                ffi::xcw_native_session_open_app_switcher(self.handle, &mut error),
+                error,
+            )
+        }
+    }
+
+    pub fn rotate_left(&self) -> Result<(), AppError> {
+        unsafe {
+            let mut error = ptr::null_mut();
+            bool_result(
+                ffi::xcw_native_session_rotate_left(self.handle, &mut error),
+                error,
+            )
+        }
+    }
+
+    pub fn rotate_right(&self) -> Result<(), AppError> {
+        unsafe {
+            let mut error = ptr::null_mut();
+            bool_result(
+                ffi::xcw_native_session_rotate_right(self.handle, &mut error),
+                error,
+            )
+        }
+    }
 }
 
 impl Drop for NativeSession {
diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs
@@ -202,6 +202,22 @@ unsafe extern "C" {
         modifiers: u32,
         error_message: *mut *mut c_char,
     ) -> bool;
+    pub fn xcw_native_session_press_home(
+        handle: *mut c_void,
+        error_message: *mut *mut c_char,
+    ) -> bool;
+    pub fn xcw_native_session_open_app_switcher(
+        handle: *mut c_void,
+        error_message: *mut *mut c_char,
+    ) -> bool;
+    pub fn xcw_native_session_rotate_right(
+        handle: *mut c_void,
+        error_message: *mut *mut c_char,
+    ) -> bool;
+    pub fn xcw_native_session_rotate_left(
+        handle: *mut c_void,
+        error_message: *mut *mut c_char,
+    ) -> bool;
 
     pub fn xcw_native_free_string(value: *mut c_char);
     pub fn xcw_native_free_bytes(bytes: xcw_native_owned_bytes);
diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs
@@ -182,6 +182,22 @@ impl SimulatorSession {
         self.inner.native.send_key(key_code, modifiers)
     }
 
+    pub fn press_home(&self) -> Result<(), AppError> {
+        self.inner.native.press_home()
+    }
+
+    pub fn open_app_switcher(&self) -> Result<(), AppError> {
+        self.inner.native.open_app_switcher()
+    }
+
+    pub fn rotate_left(&self) -> Result<(), AppError> {
+        self.inner.native.rotate_left()
+    }
+
+    pub fn rotate_right(&self) -> Result<(), AppError> {
+        self.inner.native.rotate_right()
+    }
+
     pub fn snapshot(&self) -> serde_json::Value {
         serde_json::json!({
             "displayReady": self.inner.display_ready.load(Ordering::Relaxed),
diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs
