Stabilize Android WebRTC readiness

Author: DjDeveloperr

client/src/features/stream/streamWorkerClient.ts

@@ -353,6 +353,13 @@ interface WebCodecsVideoDecoderConstructor {
   }>;
 }
 
+interface WebRtcAnswerPayload extends RTCSessionDescriptionInit {
+  video?: {
+    height?: number;
+    width?: number;
+  };
+}
+
 interface PendingVideoFrame {
   frame: WebCodecsVideoFrame;
   sequence: number | null;
@@ -1532,7 +1539,7 @@ class WebRtcStreamClient implements StreamClientBackend {
       target,
       localDescription,
     );
-    const answer = (await response.json()) as RTCSessionDescriptionInit;
+    const answer = (await response.json()) as WebRtcAnswerPayload;
     if (generation !== this.connectGeneration) {
       return;
     }
@@ -1541,6 +1548,15 @@ class WebRtcStreamClient implements StreamClientBackend {
     );
     this.postDiagnostics(target, `${options.detailPrefix}-answer`);
     await peerConnection.setRemoteDescription(answer);
+    if (
+      typeof answer.video?.width === "number" &&
+      typeof answer.video?.height === "number" &&
+      answer.video.width > 0 &&
+      answer.video.height > 0
+    ) {
+      this.syncCanvasSize(answer.video.width, answer.video.height);
+      this.reportVideoConfig(answer.video.width, answer.video.height);
+    }
   }
 
   destroy() {
@@ -1687,10 +1703,14 @@ class WebRtcStreamClient implements StreamClientBackend {
           return;
         }
         const now = performance.now();
-        const hasRenderedFrame = this.stats.renderedFrames > 0;
+        const hasMediaProgress =
+          this.hasRenderedFrame ||
+          this.stats.renderedFrames > 0 ||
+          this.stats.decodedFrames > 0 ||
+          this.stats.receivedPackets > 0;
         const frameAgeMs =
           this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity;
-        if (!hasRenderedFrame) {
+        if (!hasMediaProgress) {
           this.handleConnectionError(
             target,
             generation,
@@ -1699,6 +1719,10 @@ class WebRtcStreamClient implements StreamClientBackend {
           );
           return;
         }
+        if (!this.hasRenderedFrame) {
+          this.scheduleFrameWatchdog(target, generation);
+          return;
+        }
         if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) {
           this.sendControl({ snapshot: true, type: "streamControl" });
           this.scheduleFrameWatchdog(target, generation);

server/src/transport/webrtc.rs

@@ -89,6 +89,14 @@ pub struct WebRtcAnswerPayload {
     pub sdp: String,
     #[serde(rename = "type")]
     pub kind: String,
+    pub video: WebRtcVideoMetadata,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct WebRtcVideoMetadata {
+    pub width: u32,
+    pub height: u32,
 }
 
 #[derive(Debug, Clone, Serialize)]
@@ -289,6 +297,8 @@ pub async fn create_answer(
         summarize_sdp_candidate_types(&local_description.sdp)
     );
 
+    let first_frame_width = first_frame.width;
+    let first_frame_height = first_frame.height;
     let (cancellation_token, cancellation) =
         register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true);
     tokio::spawn(
@@ -309,6 +319,10 @@ pub async fn create_answer(
     Ok(WebRtcAnswerPayload {
         sdp: local_description.sdp,
         kind: "answer".to_owned(),
+        video: WebRtcVideoMetadata {
+            width: first_frame_width,
+            height: first_frame_height,
+        },
     })
 }