Tighten Android local stream behavior

DjDeveloperr · DjDeveloperr · commit e0914b7c3610 · 2026-05-12T01:01:52.000-04:00
diff --git a/README.md b/README.md
@@ -35,8 +35,8 @@ view inside the editor.
 
 ## Features
 
-- Local iOS Simulator and Android emulator video over browser-native WebRTC H.264 with H.264 WebSocket fallback
-- Android emulator frames are sourced from emulator gRPC and encoded through macOS VideoToolbox
+- Local iOS Simulator video over browser-native WebRTC H.264 with H.264 WebSocket fallback
+- Android emulator frames are sourced from emulator gRPC; loopback browsers use raw RGBA over WebRTC, and non-loopback browsers use VideoToolbox-encoded H.264
 - Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI
 - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents
 - CoreSimulator chrome asset rendering for device bezels
diff --git a/client/src/features/stream/streamWorkerClient.test.ts b/client/src/features/stream/streamWorkerClient.test.ts
@@ -4,6 +4,7 @@ import {
   buildStreamTarget,
   initialStreamBackend,
   preferredStreamBackend,
+  shouldUseLocalAndroidRgbaWebRtc,
 } from "./streamWorkerClient";
 
 describe("streamWorkerClient", () => {
@@ -61,4 +62,63 @@ describe("streamWorkerClient", () => {
       ).RTCPeerConnection = previousPeerConnection;
     }
   });
+
+  it("uses RGBA WebRTC transport for local loopback Android streams", () => {
+    const previousWindow = globalThis.window;
+    Object.defineProperty(globalThis, "window", {
+      configurable: true,
+      value: { location: { hostname: "127.0.0.1", search: "" } },
+    });
+
+    try {
+      expect(
+        shouldUseLocalAndroidRgbaWebRtc(
+          buildStreamTarget("android:Pixel_8", {
+            platform: "android-emulator",
+            transport: "auto",
+          }),
+        ),
+      ).toBe(true);
+    } finally {
+      Object.defineProperty(globalThis, "window", {
+        configurable: true,
+        value: previousWindow,
+      });
+    }
+  });
+
+  it("keeps Android RGBA disabled for h264 or remote streams", () => {
+    const previousWindow = globalThis.window;
+    const location = { hostname: "127.0.0.1", search: "?stream=h264" };
+    Object.defineProperty(globalThis, "window", {
+      configurable: true,
+      value: { location },
+    });
+
+    try {
+      expect(
+        shouldUseLocalAndroidRgbaWebRtc(
+          buildStreamTarget("android:Pixel_8", {
+            platform: "android-emulator",
+            transport: "auto",
+          }),
+        ),
+      ).toBe(false);
+      location.search = "";
+      expect(
+        shouldUseLocalAndroidRgbaWebRtc(
+          buildStreamTarget("android:Pixel_8", {
+            platform: "android-emulator",
+            remote: true,
+            transport: "auto",
+          }),
+        ),
+      ).toBe(false);
+    } finally {
+      Object.defineProperty(globalThis, "window", {
+        configurable: true,
+        value: previousWindow,
+      });
+    }
+  });
 });
diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts
@@ -2684,12 +2684,20 @@ function shouldUseRemoteH264AutoProfile(
   return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname);
 }
 
-function shouldUseLocalAndroidRgbaWebRtc(target: StreamConnectTarget): boolean {
+export function shouldUseLocalAndroidRgbaWebRtc(
+  target: StreamConnectTarget,
+): boolean {
   const stream = new URLSearchParams(window.location.search).get("stream");
+  if (
+    target.transport === "h264" ||
+    stream === "h264" ||
+    stream === "h264-ws"
+  ) {
+    return false;
+  }
   return (
     !target.remote &&
     isLoopbackHost(window.location.hostname) &&
-    (stream === "rgba" || stream === "webrtc-rgba") &&
     (target.platform === "android-emulator" ||
       target.udid.startsWith("android:"))
   );
diff --git a/docs/api/rest.md b/docs/api/rest.md
@@ -207,16 +207,10 @@ and the server responds with an SDP answer for a receive-only H.264 video track:
 }
 ```
 
-The endpoint requires the selected device stream to produce H.264-compatible
-samples. For iOS, samples come from the native simulator display session. For
-Android, the server reads raw frames from the emulator gRPC `streamScreenshot`
-API, encodes them through VideoToolbox, and writes the resulting H.264 samples
-to the same WebRTC video track.
-
-Loopback Android clients may pass `"transport": "rgba"` in the offer payload.
-That keeps the same WebRTC endpoint and control channels, but sends raw RGBA
-frames over the `simdeck-rgba` data channel instead of creating a media track.
-Non-loopback Android clients use encoded H.264.
+For iOS, samples come from the native simulator display session and are sent as
+an H.264 media track. Android loopback clients use the same endpoint and control
+channels, but receive raw RGBA frames over the `simdeck-rgba` data channel.
+Non-loopback Android clients receive VideoToolbox-encoded H.264.
 
 The browser also opens `simdeck-control` and `simdeck-telemetry` data channels.
 In addition to input messages, clients can request a keyframe or tune the
diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md
@@ -39,15 +39,15 @@ client/
     └── styles/
 ```
 
-| Folder                    | Responsibility                                                               |
-| ------------------------- | ---------------------------------------------------------------------------- |
-| `api/`                    | Typed wrappers around the SimDeck REST API and shared TypeScript types.      |
-| `features/simulators/`    | Sidebar list of simulators plus boot/shutdown affordances.                   |
-| `features/viewport/`      | Frame canvas, chrome compositing, hit testing.                               |
-| `features/stream/`        | WebRTC H.264 client for iOS and Android, receiver stats, and frame plumbing. |
-| `features/input/`         | Touch / keyboard / hardware-button affordances.                              |
-| `features/accessibility/` | Accessibility tree pane and source switcher.                                 |
-| `features/toolbar/`       | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh).         |
+| Folder                    | Responsibility                                                                       |
+| ------------------------- | ------------------------------------------------------------------------------------ |
+| `api/`                    | Typed wrappers around the SimDeck REST API and shared TypeScript types.              |
+| `features/simulators/`    | Sidebar list of simulators plus boot/shutdown affordances.                           |
+| `features/viewport/`      | Frame canvas, chrome compositing, hit testing.                                       |
+| `features/stream/`        | WebRTC media/RGBA stream client, H.264 fallback, receiver stats, and frame plumbing. |
+| `features/input/`         | Touch / keyboard / hardware-button affordances.                                      |
+| `features/accessibility/` | Accessibility tree pane and source switcher.                                         |
+| `features/toolbar/`       | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh).                 |
 
 ## Bootstrap flow
 
@@ -56,7 +56,7 @@ client/
 3. `AppShell` calls `GET /api/health` to learn the active encoder mode.
 4. The simulator sidebar fetches `GET /api/simulators` and renders the list.
 5. Selecting a device posts an SDP offer to `/api/simulators/<udid>/webrtc/offer`.
-6. The browser renders the H.264 video track through native WebRTC playback. Android emulator frames are sourced from emulator gRPC on the server and encoded through VideoToolbox before WebRTC delivery.
+6. The browser renders the WebRTC stream. iOS uses an H.264 media track; Android loopback clients use a raw RGBA data channel, while non-loopback Android clients use VideoToolbox-encoded H.264.
 7. Touch and key events round-trip through `POST /api/simulators/<udid>/touch` and `/key`.
 
 ## Dev workflow
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
@@ -2963,7 +2963,9 @@ async fn accessibility_point(
             android.accessibility_tree(&udid, None)
         })
         .await?;
-        return Ok(json(snapshot));
+        return Ok(json(accessibility_point_snapshot(
+            &snapshot, query.x, query.y,
+        )?));
     }
     let snapshot = accessibility_snapshot(state, udid, Some((query.x, query.y)), None).await?;
     Ok(json(snapshot))
@@ -3683,6 +3685,77 @@ fn normalize_screen_point_from_snapshot(
     Ok(((x / width).clamp(0.0, 1.0), (y / height).clamp(0.0, 1.0)))
 }
 
+fn accessibility_point_snapshot(snapshot: &Value, x: f64, y: f64) -> Result<Value, AppError> {
+    let roots = snapshot
+        .get("roots")
+        .and_then(Value::as_array)
+        .ok_or_else(|| AppError::not_found("Accessibility snapshot does not contain roots."))?;
+    let node = roots
+        .iter()
+        .rev()
+        .find_map(|root| deepest_node_at_point(root, x, y))
+        .ok_or_else(|| AppError::not_found("No accessibility element contains the point."))?;
+    let mut node = node.clone();
+    if let Some(object) = node.as_object_mut() {
+        object.remove("children");
+    }
+
+    let mut response = Map::new();
+    for key in [
+        "source",
+        "availableSources",
+        "requestedSource",
+        "fallbackReason",
+        "inspector",
+        "includeHidden",
+    ] {
+        if let Some(value) = snapshot.get(key) {
+            response.insert(key.to_owned(), value.clone());
+        }
+    }
+    response.insert("roots".to_owned(), Value::Array(vec![node]));
+    Ok(Value::Object(response))
+}
+
+fn deepest_node_at_point(node: &Value, x: f64, y: f64) -> Option<&Value> {
+    let has_frame = node
+        .get("frame")
+        .or_else(|| node.get("frameInScreen"))
+        .is_some();
+    if has_frame && !node_frame_contains_point(node, x, y).unwrap_or(false) {
+        return None;
+    }
+    for child in node
+        .get("children")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .rev()
+    {
+        if let Some(found) = deepest_node_at_point(child, x, y) {
+            return Some(found);
+        }
+    }
+    has_frame.then_some(node)
+}
+
+fn node_frame_contains_point(node: &Value, x: f64, y: f64) -> Result<bool, AppError> {
+    let frame = node
+        .get("frame")
+        .or_else(|| node.get("frameInScreen"))
+        .ok_or_else(|| AppError::not_found("Accessibility node does not expose a frame."))?;
+    let frame_x = number_field(frame, "x")?;
+    let frame_y = number_field(frame, "y")?;
+    let width = number_field(frame, "width")?;
+    let height = number_field(frame, "height")?;
+    Ok(width > 0.0
+        && height > 0.0
+        && x >= frame_x
+        && y >= frame_y
+        && x < frame_x + width
+        && y < frame_y + height)
+}
+
 fn number_field(value: &Value, field: &str) -> Result<f64, AppError> {
     value
         .get(field)
@@ -5328,10 +5401,11 @@ async fn simulator_payload(state: AppState, udid: String) -> Result<Json<Value>,
 #[cfg(test)]
 mod tests {
     use super::{
-        attach_tree_metadata, available_sources_for_snapshot, best_inspector_session,
-        compact_accessibility_snapshot, element_matches_selector, first_matching_element,
-        inspector_available_sources, inspector_metadata, inspector_session_from_published,
-        inspector_session_score, is_inspector_agent_transport_path, normalize_inspector_node,
+        accessibility_point_snapshot, attach_tree_metadata, available_sources_for_snapshot,
+        best_inspector_session, compact_accessibility_snapshot, element_matches_selector,
+        first_matching_element, inspector_available_sources, inspector_metadata,
+        inspector_session_from_published, inspector_session_score,
+        is_inspector_agent_transport_path, normalize_inspector_node,
         normalize_screen_point_from_snapshot, normalized_gesture_coordinates,
         parse_lsof_tcp_listener, resolved_stream_quality_limits, split_filter_values,
         stream_quality_profile, suppress_native_ax_translation_error, tap_point_from_snapshot,
@@ -5431,6 +5505,35 @@ mod tests {
         assert_eq!(point, (1.0, 0.0));
     }
 
+    #[test]
+    fn accessibility_point_snapshot_returns_deepest_node() {
+        let snapshot = json!({
+            "source": "android-uiautomator",
+            "availableSources": ["android-uiautomator"],
+            "roots": [{
+                "type": "FrameLayout",
+                "frame": { "x": 0.0, "y": 0.0, "width": 400.0, "height": 800.0 },
+                "children": [{
+                    "type": "ViewGroup",
+                    "AXIdentifier": "container",
+                    "frame": { "x": 0.0, "y": 100.0, "width": 400.0, "height": 300.0 },
+                    "children": [{
+                        "type": "Button",
+                        "AXIdentifier": "child-button",
+                        "frame": { "x": 120.0, "y": 140.0, "width": 80.0, "height": 60.0 },
+                        "children": []
+                    }]
+                }]
+            }]
+        });
+
+        let point = accessibility_point_snapshot(&snapshot, 150.0, 160.0).unwrap();
+
+        assert_eq!(point["source"], "android-uiautomator");
+        assert_eq!(point["roots"][0]["AXIdentifier"], "child-button");
+        assert!(point["roots"][0].get("children").is_none());
+    }
+
     #[test]
     fn gesture_presets_clamp_delta_and_reject_unknown_names() {
         assert_eq!(
diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs
@@ -164,7 +164,7 @@ pub async fn create_answer(
                 state.android.clone(),
                 state.metrics.clone(),
                 udid.clone(),
-                android_webrtc_max_edge(),
+                None,
                 true,
             )
             .await?,
@@ -1360,9 +1360,6 @@ impl AndroidWebRtcSource {
                         match frame {
                             Ok(Some(frame)) => {
                                 let frame = Arc::new(frame);
-                                if let Some(inner) = reader_inner.upgrade() {
-                                    let _ = inner.raw_sender.send(frame.clone());
-                                }
                                 *reader_latest_frame.lock().unwrap() = Some(frame);
                             }
                             Ok(None) => break,
@@ -1679,14 +1676,6 @@ unsafe fn take_native_error(raw: *mut i8) -> Option<AppError> {
     Some(AppError::native(message))
 }
 
-fn android_webrtc_max_edge() -> Option<u32> {
-    std::env::var("SIMDECK_ANDROID_WEBRTC_MAX_EDGE")
-        .ok()
-        .and_then(|value| value.parse::<u32>().ok())
-        .filter(|value| *value > 0)
-        .map(|value| value.clamp(360, 4096))
-}
-
 fn android_webrtc_frame_interval() -> Duration {
     Duration::from_micros(1_000_000 / ANDROID_WEBRTC_FPS)
 }
diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md
@@ -59,9 +59,9 @@ Build apps with project tooling.
 
 Android devices use IDs like `android:Pixel_8_API_36`. `simdeck list` discovers
 AVDs from the Android SDK, `boot` starts `emulator -avd ... -no-window`, and
-live browser viewing uses the same WebRTC H.264 endpoint as iOS. Android frames
-come from emulator gRPC and are encoded through VideoToolbox. `simdeck stream`
-is still iOS-only.
+live browser viewing uses WebRTC. Local loopback Android viewing sends raw RGBA
+frames over a WebRTC data channel; non-loopback Android viewing uses
+VideoToolbox-encoded H.264. `simdeck stream` is still iOS-only.
 
 ## Fast Agent Inspection
 
