Skip to content

Commit 1986f2f

Browse files
DjDeveloperrDjDeveloperr
committed
Reduce touch latency and align Safari H264
1 parent 350ea44 commit 1986f2f

3 files changed

Lines changed: 123 additions & 22 deletions

File tree

‎client/src/app/AppShell.tsx‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,10 @@ import { useKeyboardInput } from "../features/input/useKeyboardInput";
3939
import { usePointerInput } from "../features/input/usePointerInput";
4040
import { simulatorRuntimeLabel } from "../features/simulators/simulatorDisplay";
4141
import { useSimulatorList } from "../features/simulators/useSimulatorList";
42-
import { sendWebRtcControlMessage } from "../features/stream/streamWorkerClient";
42+
import {
43+
sendWebRtcControlMessage,
44+
sendWebRtcRealtimeControlMessage,
45+
} from "../features/stream/streamWorkerClient";
4346
import { useLiveStream } from "../features/stream/useLiveStream";
4447
import { DebugPanel } from "../features/toolbar/DebugPanel";
4548
import { Toolbar } from "../features/toolbar/Toolbar";
@@ -843,7 +846,11 @@ export function AppShell({
843846
setAccessibilitySelectedId("");
844847
setAccessibilityHoveredId(null);
845848
}
846-
sendControl(selectedSimulator.udid, { type: "touch", ...coords, phase });
849+
sendControl(
850+
selectedSimulator.udid,
851+
{ type: "touch", ...coords, phase },
852+
{ realtime: phase === "moved" },
853+
);
847854
},
848855
onTouchPreview: showTouchIndicator,
849856
pan,
@@ -995,10 +1002,20 @@ export function AppShell({
9951002
return state;
9961003
}, []);
9971004

998-
function sendControl(udid: string, message: ControlMessage): boolean {
1005+
function sendControl(
1006+
udid: string,
1007+
message: ControlMessage,
1008+
options: { realtime?: boolean } = {},
1009+
): boolean {
9991010
setLocalError("");
10001011
const encoded = JSON.stringify(message);
1001-
if (sendWebRtcControlMessage(encoded)) {
1012+
const sent = options.realtime
1013+
? sendWebRtcRealtimeControlMessage(encoded)
1014+
: sendWebRtcControlMessage(encoded);
1015+
if (sent) {
1016+
return true;
1017+
}
1018+
if (options.realtime) {
10021019
return true;
10031020
}
10041021
const state = ensureControlSocket(udid);

‎client/src/features/stream/streamWorkerClient.ts‎

Lines changed: 90 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,30 +10,53 @@ import type {
1010

1111
const HAVE_CURRENT_DATA = 2;
1212
const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control";
13+
const WEBRTC_REALTIME_INPUT_CHANNEL_LABEL = "simdeck-input";
14+
const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry";
1315
const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000;
1416
const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000;
1517
const WEBRTC_REMOTE_DISCONNECTED_GRACE_MS = 3000;
18+
const WEBRTC_REALTIME_INPUT_BUFFER_LIMIT = 1024;
1619

1720
let activeWebRtcControlChannel: RTCDataChannel | null = null;
21+
let activeWebRtcRealtimeInputChannel: RTCDataChannel | null = null;
22+
let activeWebRtcTelemetryChannel: RTCDataChannel | null = null;
1823
let activeStreamClient: StreamWorkerClient | null = null;
1924

2025
export type StreamBackend = "webrtc";
2126

2227
export function sendWebRtcControlMessage(encoded: string): boolean {
23-
if (activeWebRtcControlChannel?.readyState !== "open") {
28+
return sendDataChannelMessage(activeWebRtcControlChannel, encoded);
29+
}
30+
31+
export function sendWebRtcRealtimeControlMessage(encoded: string): boolean {
32+
if (activeWebRtcRealtimeInputChannel?.readyState !== "open") {
2433
return false;
2534
}
26-
activeWebRtcControlChannel.send(encoded);
35+
if (
36+
activeWebRtcRealtimeInputChannel.bufferedAmount >
37+
WEBRTC_REALTIME_INPUT_BUFFER_LIMIT
38+
) {
39+
return true;
40+
}
41+
activeWebRtcRealtimeInputChannel.send(encoded);
2742
return true;
2843
}
2944

3045
export function sendWebRtcClientStats(stats: unknown): boolean {
31-
if (activeWebRtcControlChannel?.readyState !== "open") {
32-
return false;
33-
}
34-
activeWebRtcControlChannel.send(
46+
return sendDataChannelMessage(
47+
activeWebRtcTelemetryChannel,
3548
JSON.stringify({ stats, type: "clientStats" }),
3649
);
50+
}
51+
52+
function sendDataChannelMessage(
53+
channel: RTCDataChannel | null,
54+
encoded: string,
55+
): boolean {
56+
if (channel?.readyState !== "open") {
57+
return false;
58+
}
59+
channel.send(encoded);
3760
return true;
3861
}
3962

@@ -67,9 +90,11 @@ class WebRtcStreamClient implements StreamClientBackend {
6790
private lastVideoFrameAt = 0;
6891
private peerConnection: RTCPeerConnection | null = null;
6992
private reconnectTimeout = 0;
93+
private realtimeInputChannel: RTCDataChannel | null = null;
7094
private remoteMode = false;
7195
private reportedVideoConfig = false;
7296
private shouldReconnect = false;
97+
private telemetryChannel: RTCDataChannel | null = null;
7398
private stats: StreamStats = createEmptyStreamStats();
7499
private video: HTMLVideoElement | null = null;
75100
private videoFrameCallback = 0;
@@ -134,6 +159,34 @@ class WebRtcStreamClient implements StreamClientBackend {
134159
activeWebRtcControlChannel = null;
135160
}
136161
});
162+
const realtimeInputChannel = peerConnection.createDataChannel(
163+
WEBRTC_REALTIME_INPUT_CHANNEL_LABEL,
164+
{
165+
maxRetransmits: 0,
166+
ordered: false,
167+
},
168+
);
169+
this.realtimeInputChannel = realtimeInputChannel;
170+
activeWebRtcRealtimeInputChannel = realtimeInputChannel;
171+
realtimeInputChannel.addEventListener("close", () => {
172+
if (activeWebRtcRealtimeInputChannel === realtimeInputChannel) {
173+
activeWebRtcRealtimeInputChannel = null;
174+
}
175+
});
176+
const telemetryChannel = peerConnection.createDataChannel(
177+
WEBRTC_TELEMETRY_CHANNEL_LABEL,
178+
{
179+
maxRetransmits: 0,
180+
ordered: false,
181+
},
182+
);
183+
this.telemetryChannel = telemetryChannel;
184+
activeWebRtcTelemetryChannel = telemetryChannel;
185+
telemetryChannel.addEventListener("close", () => {
186+
if (activeWebRtcTelemetryChannel === telemetryChannel) {
187+
activeWebRtcTelemetryChannel = null;
188+
}
189+
});
137190

138191
peerConnection.ontrack = (event) => {
139192
if (generation !== this.connectGeneration) {
@@ -223,7 +276,7 @@ class WebRtcStreamClient implements StreamClientBackend {
223276
}
224277
};
225278

226-
const offer = await peerConnection.createOffer();
279+
const offer = safariBaselineH264Offer(await peerConnection.createOffer());
227280
if (generation !== this.connectGeneration) {
228281
return;
229282
}
@@ -292,6 +345,16 @@ class WebRtcStreamClient implements StreamClientBackend {
292345
activeWebRtcControlChannel = null;
293346
}
294347
this.controlChannel = null;
348+
this.realtimeInputChannel?.close();
349+
if (activeWebRtcRealtimeInputChannel === this.realtimeInputChannel) {
350+
activeWebRtcRealtimeInputChannel = null;
351+
}
352+
this.realtimeInputChannel = null;
353+
this.telemetryChannel?.close();
354+
if (activeWebRtcTelemetryChannel === this.telemetryChannel) {
355+
activeWebRtcTelemetryChannel = null;
356+
}
357+
this.telemetryChannel = null;
295358
this.peerConnection?.close();
296359
this.peerConnection = null;
297360
}
@@ -716,6 +779,26 @@ function configureReceiverCodecPreferences(transceiver: RTCRtpTransceiver) {
716779
]);
717780
}
718781

782+
function safariBaselineH264Offer(
783+
offer: RTCSessionDescriptionInit,
784+
): RTCSessionDescriptionInit {
785+
if (!isSafariBrowser() || !offer.sdp) {
786+
return offer;
787+
}
788+
return {
789+
...offer,
790+
sdp: offer.sdp.replace(
791+
/(a=fmtp:\d+ .*profile-level-id=)[0-9a-fA-F]{6}/g,
792+
"$142e01f",
793+
),
794+
};
795+
}
796+
797+
function isSafariBrowser(): boolean {
798+
const ua = navigator.userAgent;
799+
return /Safari\//.test(ua) && !/Chrome\/|Chromium\/|CriOS\/|FxiOS\//.test(ua);
800+
}
801+
719802
function iceServers(health?: HealthResponse | null): RTCIceServer[] {
720803
const params = new URLSearchParams(window.location.search);
721804
const queryValue = params.get("iceServers");

‎server/src/transport/webrtc.rs‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ use webrtc::track::track_local::TrackLocalWriter;
3636
const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1];
3737
const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302";
3838
const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control";
39+
const WEBRTC_REALTIME_INPUT_CHANNEL_LABEL: &str = "simdeck-input";
40+
const WEBRTC_TELEMETRY_CHANNEL_LABEL: &str = "simdeck-telemetry";
3941
const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150);
4042
const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3;
4143
const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16);
@@ -345,7 +347,13 @@ fn register_control_data_channel(
345347
let state = state.clone();
346348
let udid = udid.clone();
347349
Box::pin(async move {
348-
if channel.label() != WEBRTC_CONTROL_CHANNEL_LABEL {
350+
let label = channel.label();
351+
if !matches!(
352+
label.as_ref(),
353+
WEBRTC_CONTROL_CHANNEL_LABEL
354+
| WEBRTC_REALTIME_INPUT_CHANNEL_LABEL
355+
| WEBRTC_TELEMETRY_CHANNEL_LABEL
356+
) {
349357
return;
350358
}
351359
attach_control_data_channel(channel, session, state, udid);
@@ -431,14 +439,7 @@ fn h264_sdp_fmtp_line(codec: &str, offer_sdp: &str) -> String {
431439
.map(|value| value.to_ascii_lowercase());
432440
let offered_profile_level_ids = offer_h264_profile_level_ids(offer_sdp);
433441
let profile_level_id = codec_profile_level_id
434-
.as_deref()
435-
.filter(|codec_profile| {
436-
offered_profile_level_ids.is_empty()
437-
|| offered_profile_level_ids
438-
.iter()
439-
.any(|offered_profile| offered_profile == *codec_profile)
440-
})
441-
.map(str::to_owned)
442+
.clone()
442443
.or_else(|| offered_profile_level_ids.first().cloned())
443444
.unwrap_or_else(|| "42e01f".to_owned());
444445
format!("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id={profile_level_id}")
@@ -1196,8 +1197,8 @@ mod tests {
11961197
assert!(h264_sdp_fmtp_line("avc1.42e01f", "").contains("profile-level-id=42e01f"));
11971198
assert!(h264_sdp_fmtp_line("h264", "").contains("profile-level-id=42e01f"));
11981199
assert!(h264_sdp_fmtp_line(
1199-
"avc1.640034",
1200-
"a=rtpmap:99 H264/90000\r\na=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\n"
1200+
"avc1.42e01f",
1201+
"a=rtpmap:99 H264/90000\r\na=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f\r\n"
12011202
)
12021203
.contains("profile-level-id=42e01f"));
12031204
}

0 commit comments

Comments
 (0)