Skip to content

Commit 2da61ae

Browse files
DjDeveloperrDjDeveloperr
committed
Raise local stream cap to 120 fps
1 parent 125fcca commit 2da61ae

12 files changed

Lines changed: 183 additions & 50 deletions

File tree

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ more important than full-resolution smoothness:
116116
simdeck daemon start --video-codec software --low-latency
117117
```
118118

119+
Local browser streams default to 60 fps. On high-refresh local displays, opt in
120+
to a paced hardware stream up to 120 fps:
121+
122+
```sh
123+
simdeck daemon restart --local-stream-fps 120
124+
```
125+
119126
Restart the CoreSimulator service layer when `simctl` reports a stale service
120127
version or the live display gets stuck before the first frame:
121128

‎cli/XCWH264Encoder.m‎

Lines changed: 67 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
static const int32_t XCWTargetRealTimeFrameRate = 60;
1515
static const int32_t XCWTargetRealtimeHardwareFrameRate = 30;
1616
static const int32_t XCWTargetSoftwareFrameRate = 60;
17+
static const int32_t XCWMinimumLocalStreamFrameRate = 15;
18+
static const int32_t XCWMaximumLocalStreamFrameRate = 120;
1719
static const int32_t XCWTargetLowLatencySoftwareFrameRate = 15;
1820
static const NSUInteger XCWMaximumInFlightFrames = 2;
1921
static const int32_t XCWMinimumAverageBitRate = 18000000;
@@ -137,6 +139,22 @@ static uint64_t XCWRealtimeMaximumFrameIntervalUs(void) {
137139
return MAX(XCWRealtimeFrameIntervalUs() * 2, XCWRealtimeFrameIntervalUs());
138140
}
139141

142+
static int32_t XCWLocalStreamTargetFrameRate(void) {
143+
return XCWIntFromEnvironment(@"SIMDECK_LOCAL_STREAM_FPS",
144+
XCWTargetRealTimeFrameRate,
145+
XCWMinimumLocalStreamFrameRate,
146+
XCWMaximumLocalStreamFrameRate);
147+
}
148+
149+
static uint64_t XCWLocalStreamFrameIntervalUs(void) {
150+
int32_t fps = MAX(1, XCWLocalStreamTargetFrameRate());
151+
return (uint64_t)llround(1000000.0 / (double)fps);
152+
}
153+
154+
static uint64_t XCWLocalStreamMaximumFrameIntervalUs(void) {
155+
return MAX(XCWSoftwareMaximumFrameIntervalUs, XCWLocalStreamFrameIntervalUs());
156+
}
157+
140158
static int64_t XCWRealtimeBitsPerPixelBudgetValue(void) {
141159
return XCWInt64FromEnvironment(@"SIMDECK_REALTIME_BITS_PER_PIXEL",
142160
XCWRealtimeBitsPerPixelBudget,
@@ -490,10 +508,10 @@ @implementation XCWH264Encoder {
490508
uint64_t _lastSoftwareSubmissionUs;
491509
NSUInteger _softwarePacedFrameCount;
492510
NSUInteger _softwareHealthyFrameCount;
493-
uint64_t _realtimeHardwareFrameIntervalUs;
494-
uint64_t _lastRealtimeHardwareSubmissionUs;
495-
NSUInteger _realtimeHardwarePacedFrameCount;
496-
NSUInteger _realtimeHardwareHealthyFrameCount;
511+
uint64_t _hardwareFrameIntervalUs;
512+
uint64_t _lastHardwareSubmissionUs;
513+
NSUInteger _hardwarePacedFrameCount;
514+
NSUInteger _hardwareHealthyFrameCount;
497515
NSString *_selectedEncoderID;
498516
NSInteger _lastSessionStatus;
499517
NSInteger _lastPrepareStatus;
@@ -518,7 +536,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler
518536
_realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode;
519537
_codecType = XCWVideoCodecTypeForMode(_encoderMode);
520538
_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
521-
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
539+
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
522540
return self;
523541
}
524542

@@ -568,9 +586,9 @@ - (void)reconfigureForStreamQualityChange {
568586
self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
569587
self->_softwarePacedFrameCount = 0;
570588
self->_softwareHealthyFrameCount = 0;
571-
self->_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
572-
self->_realtimeHardwarePacedFrameCount = 0;
573-
self->_realtimeHardwareHealthyFrameCount = 0;
589+
self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
590+
self->_hardwarePacedFrameCount = 0;
591+
self->_hardwareHealthyFrameCount = 0;
574592
});
575593
}
576594

@@ -597,9 +615,13 @@ - (NSDictionary *)statsRepresentation {
597615
@"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs),
598616
@"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0),
599617
@"softwarePacedFrames": @(self->_softwarePacedFrameCount),
600-
@"realtimeHardwareFrameIntervalUs": @(self->_realtimeHardwareFrameIntervalUs),
601-
@"realtimeHardwareTargetFps": @(self->_realtimeHardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_realtimeHardwareFrameIntervalUs) : 0.0),
602-
@"realtimeHardwarePacedFrames": @(self->_realtimeHardwarePacedFrameCount),
618+
@"localStreamTargetFps": @(XCWLocalStreamTargetFrameRate()),
619+
@"hardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs),
620+
@"hardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0),
621+
@"hardwarePacedFrames": @(self->_hardwarePacedFrameCount),
622+
@"realtimeHardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs),
623+
@"realtimeHardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0),
624+
@"realtimeHardwarePacedFrames": @(self->_hardwarePacedFrameCount),
603625
@"transportCodec": XCWCodecName(self->_codecType),
604626
@"encoderMode": XCWVideoEncoderModeName(self->_encoderMode),
605627
@"lowLatencyMode": @(self->_lowLatencyMode),
@@ -665,15 +687,15 @@ - (NSUInteger)softwareHealthyFrameWindowLocked {
665687
}
666688

667689
- (uint64_t)minimumHardwareFrameIntervalUsLocked {
668-
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWSoftwareMinimumFrameIntervalUs;
690+
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs();
669691
}
670692

671693
- (uint64_t)initialHardwareFrameIntervalUsLocked {
672-
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWSoftwareInitialFrameIntervalUs;
694+
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs();
673695
}
674696

675697
- (uint64_t)maximumHardwareFrameIntervalUsLocked {
676-
return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWSoftwareMaximumFrameIntervalUs;
698+
return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWLocalStreamMaximumFrameIntervalUs();
677699
}
678700

679701
- (int32_t)expectedFrameRateLocked {
@@ -686,24 +708,24 @@ - (int32_t)expectedFrameRateLocked {
686708
if (_realtimeStreamMode) {
687709
return XCWRealtimeTargetFrameRate();
688710
}
689-
return XCWTargetRealTimeFrameRate;
711+
return XCWLocalStreamTargetFrameRate();
690712
}
691713

692-
- (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs {
693-
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || _needsKeyFrame) {
714+
- (BOOL)shouldPaceHardwareFrameAtTimeUs:(uint64_t)nowUs {
715+
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || _needsKeyFrame) {
694716
return NO;
695717
}
696-
if (_realtimeHardwareFrameIntervalUs == 0) {
697-
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
718+
if (_hardwareFrameIntervalUs == 0) {
719+
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
698720
}
699-
if (_lastRealtimeHardwareSubmissionUs == 0) {
721+
if (_lastHardwareSubmissionUs == 0) {
700722
return NO;
701723
}
702-
uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0;
703-
if (elapsedUs >= _realtimeHardwareFrameIntervalUs) {
724+
uint64_t elapsedUs = nowUs >= _lastHardwareSubmissionUs ? nowUs - _lastHardwareSubmissionUs : 0;
725+
if (elapsedUs >= _hardwareFrameIntervalUs) {
704726
return NO;
705727
}
706-
_realtimeHardwarePacedFrameCount += 1;
728+
_hardwarePacedFrameCount += 1;
707729
return YES;
708730
}
709731

@@ -764,41 +786,41 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs {
764786
_softwareHealthyFrameCount = 0;
765787
}
766788

767-
- (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs {
768-
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) {
789+
- (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs {
790+
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || latencyUs == 0) {
769791
return;
770792
}
771-
if (_realtimeHardwareFrameIntervalUs == 0) {
772-
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
793+
if (_hardwareFrameIntervalUs == 0) {
794+
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
773795
}
774796

775797
uint64_t minimumIntervalUs = [self minimumHardwareFrameIntervalUsLocked];
776798
uint64_t maximumIntervalUs = [self maximumHardwareFrameIntervalUsLocked];
777-
if (latencyUs > _realtimeHardwareFrameIntervalUs) {
778-
uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs;
799+
if (latencyUs > _hardwareFrameIntervalUs) {
800+
uint64_t nextIntervalUs = _hardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs;
779801
uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs;
780802
if (nextIntervalUs < latencyBoundIntervalUs) {
781803
nextIntervalUs = latencyBoundIntervalUs;
782804
}
783-
_realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs);
784-
_realtimeHardwareHealthyFrameCount = 0;
805+
_hardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs);
806+
_hardwareHealthyFrameCount = 0;
785807
return;
786808
}
787809

788-
if (latencyUs < _realtimeHardwareFrameIntervalUs &&
789-
_realtimeHardwareFrameIntervalUs > minimumIntervalUs) {
790-
_realtimeHardwareHealthyFrameCount += 1;
791-
if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) {
792-
uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs
793-
? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs
810+
if (latencyUs < _hardwareFrameIntervalUs &&
811+
_hardwareFrameIntervalUs > minimumIntervalUs) {
812+
_hardwareHealthyFrameCount += 1;
813+
if (_hardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) {
814+
uint64_t nextIntervalUs = _hardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs
815+
? _hardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs
794816
: minimumIntervalUs;
795-
_realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs);
796-
_realtimeHardwareHealthyFrameCount = 0;
817+
_hardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs);
818+
_hardwareHealthyFrameCount = 0;
797819
}
798820
return;
799821
}
800822

801-
_realtimeHardwareHealthyFrameCount = 0;
823+
_hardwareHealthyFrameCount = 0;
802824
}
803825

804826
- (void)drainPendingFramesLocked {
@@ -839,7 +861,7 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
839861
}
840862

841863
uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
842-
if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs]) {
864+
if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceHardwareFrameAtTimeUs:nowUs]) {
843865
return YES;
844866
}
845867

@@ -885,8 +907,8 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
885907
_submittedFrameCount += 1;
886908
if (_encoderMode == XCWVideoEncoderModeH264Software) {
887909
_lastSoftwareSubmissionUs = nowUs;
888-
} else if ((_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) && _realtimeStreamMode) {
889-
_lastRealtimeHardwareSubmissionUs = nowUs;
910+
} else if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) {
911+
_lastHardwareSubmissionUs = nowUs;
890912
}
891913
_maxInFlightFrameCount = MAX(_maxInFlightFrameCount, _inFlightFrameCount);
892914
if (_encoderMode == XCWVideoEncoderModeH264Software || !_realtimeStreamMode) {
@@ -1013,7 +1035,7 @@ - (void)invalidateCompressionSessionLocked {
10131035
_timestampOriginUs = 0;
10141036
_inFlightFrameCount = 0;
10151037
_lastSoftwareSubmissionUs = 0;
1016-
_lastRealtimeHardwareSubmissionUs = 0;
1038+
_lastHardwareSubmissionUs = 0;
10171039
_hardwareAccelerated = NO;
10181040
_selectedEncoderID = nil;
10191041
[self invalidateScalingResourcesLocked];
@@ -1203,7 +1225,7 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer
12031225
uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
12041226
_latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0;
12051227
[self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs];
1206-
[self adaptRealtimeHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
1228+
[self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
12071229
}
12081230
NSString *codec = nil;
12091231
NSData *decoderConfig = nil;

‎docs/api/health.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn
2828
| `videoCodec` | Requested encoder mode. One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). |
2929
| `lowLatency` | `true` when software H.264 low-latency mode was enabled at daemon startup. |
3030
| `realtimeStream` | `true` when the WebRTC stream is configured to favor freshness and realtime pacing. |
31+
| `localStreamFps` | Local quality stream frame cap, from 15 to 120 fps. Defaults to 60. |
3132
| `streamQuality` | Active realtime quality profile and encoder limits such as `maxEdge`, `fps`, and bitrate. |
3233
| `webRtc.iceServers` | ICE servers the browser should use when creating the WebRTC peer connection. |
3334
| `webRtc.iceTransportPolicy` | Browser ICE transport policy. One of `all` or `relay`. |

‎docs/cli/commands.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ Start or reuse the project daemon and serve the browser UI.
3232
```sh
3333
simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host <host>]
3434
[--client-root <path>] [--video-codec auto|hardware|software]
35-
[--low-latency] [--stream-quality <profile>] [--open]
35+
[--low-latency] [--stream-quality <profile>]
36+
[--local-stream-fps <15-120>] [--open]
3637
```
3738

3839
`--open` opens the authenticated local URL after the daemon is ready.
@@ -61,7 +62,7 @@ Start or reuse the project daemon without opening the browser:
6162
simdeck daemon start [--port 4310] [--bind 127.0.0.1]
6263
[--advertise-host <host>] [--client-root <path>]
6364
[--video-codec auto|hardware|software] [--low-latency]
64-
[--stream-quality <profile>]
65+
[--stream-quality <profile>] [--local-stream-fps <15-120>]
6566
```
6667

6768
Output:
@@ -97,7 +98,7 @@ options as `daemon start`:
9798
simdeck daemon restart [--port 4310] [--bind 127.0.0.1]
9899
[--advertise-host <host>] [--client-root <path>]
99100
[--video-codec auto|hardware|software] [--low-latency]
100-
[--stream-quality <profile>]
101+
[--stream-quality <profile>] [--local-stream-fps <15-120>]
101102
```
102103

103104
### `daemon stop`
@@ -126,7 +127,8 @@ that starts after login and stays available.
126127
simdeck service on [--port 4310] [--bind 127.0.0.1]
127128
[--advertise-host <host>] [--client-root <path>]
128129
[--video-codec auto|hardware|software] [--low-latency]
129-
[--stream-quality <profile>] [--access-token <token>]
130+
[--stream-quality <profile>] [--local-stream-fps <15-120>]
131+
[--access-token <token>]
130132
simdeck service restart [same options as service on]
131133
simdeck service off
132134
```

‎docs/cli/flags.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas
3737
| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). |
3838
| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. |
3939
| `--stream-quality` | auto/default | Optional realtime stream quality profile: `quality`, `balanced`, `smooth`, `economy`, or `ci-software`. |
40+
| `--local-stream-fps` | `60` | Local quality stream frame cap, from 15 to 120 fps. |
4041
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
4142

4243
`studio expose` defaults to software H.264. Pass `--video-codec hardware` to

‎docs/guide/daemon.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and
6363
| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). |
6464
| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. |
6565
| `--stream-quality` | auto/default | Optional realtime stream quality profile, including `ci-software` for CI providers. |
66+
| `--local-stream-fps` | `60` | Local quality stream frame cap, from 15 to 120 fps. |
6667
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
6768

6869
Example:

‎docs/guide/video.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques
7878
A few practical guidelines:
7979

8080
- **Start on the default for local preview.** `auto` lets VideoToolbox choose without requiring the shared hardware encoder.
81+
- **Use `--local-stream-fps 120` only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher caps pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames.
8182
- **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency.
8283
- **Studio providers default to software H.264 plus `--stream-quality smooth`.** This profile uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder.
8384
- **The remote browser renders the live stream as a native `<video>` element.** The canvas remains for input geometry, but it is not in the live per-frame render path and does not preserve stale frames during reconnects.

‎server/src/api/routes.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -565,6 +565,7 @@ async fn health(State(state): State<AppState>) -> Json<Value> {
565565
"videoCodec": active_video_codec(&state.config),
566566
"lowLatency": state.config.low_latency,
567567
"realtimeStream": crate::transport::webrtc::realtime_stream_enabled(),
568+
"localStreamFps": env_u32("SIMDECK_LOCAL_STREAM_FPS", 60, 15, 120),
568569
"streamQuality": stream_quality_state(),
569570
"webRtc": {
570571
"iceServers": crate::transport::webrtc::client_ice_servers(),

0 commit comments

Comments
 (0)