Skip to content

Commit fa31058

Browse files
DjDeveloperrDjDeveloperr
committed
Add encoder overload detection
1 parent 7f18d76 commit fa31058

15 files changed

Lines changed: 273 additions & 14 deletions

File tree

‎cli/XCWH264Encoder.m‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6;
4141
static const NSUInteger XCWMaximumRealtimeInFlightFrames = 3;
4242
static const int32_t XCWRealtimeKeyFrameIntervalSeconds = 5;
43+
static const double XCWEncoderLatencyEWMAAlpha = 0.2;
44+
static const double XCWEncoderStrainedLoadPercent = 85.0;
45+
static const double XCWEncoderOverloadedLoadPercent = 105.0;
46+
static const NSUInteger XCWEncoderConsecutiveOverBudgetFrameThreshold = 3;
4347

4448
typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) {
4549
XCWVideoEncoderModeAuto,
@@ -487,6 +491,7 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)ta
487491
pixelFormat:(OSType)pixelFormat;
488492
- (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
489493
submittedAtUs:(uint64_t)submittedAtUs;
494+
- (uint64_t)activeFrameIntervalUsLocked;
490495

491496
@end
492497

@@ -520,6 +525,13 @@ @implementation XCWH264Encoder {
520525
NSUInteger _keyFrameOutputCount;
521526
NSUInteger _maxInFlightFrameCount;
522527
uint64_t _latestEncodeLatencyUs;
528+
double _averageEncodeLatencyUs;
529+
uint64_t _peakEncodeLatencyUs;
530+
NSUInteger _overBudgetFrameCount;
531+
NSUInteger _consecutiveOverBudgetFrameCount;
532+
NSUInteger _consecutiveStrainedFrameCount;
533+
NSUInteger _overloadEventCount;
534+
BOOL _wasOverloaded;
523535
uint64_t _softwareFrameIntervalUs;
524536
uint64_t _lastSoftwareSubmissionUs;
525537
NSUInteger _softwarePacedFrameCount;
@@ -621,6 +633,33 @@ - (NSDictionary *)statsRepresentation {
621633

622634
__block NSDictionary *stats = nil;
623635
dispatch_sync(_queue, ^{
636+
uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked];
637+
double latestLoadPercent = encoderBudgetUs > 0
638+
? ((double)self->_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
639+
: 0.0;
640+
double averageLoadPercent = encoderBudgetUs > 0
641+
? (self->_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
642+
: 0.0;
643+
BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent ||
644+
self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
645+
BOOL strained = overloaded ||
646+
averageLoadPercent >= XCWEncoderStrainedLoadPercent ||
647+
self->_consecutiveStrainedFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
648+
NSString *overloadState = overloaded
649+
? @"overloaded"
650+
: strained
651+
? @"strained"
652+
: @"nominal";
653+
NSString *overloadReason = @"within-budget";
654+
if (overloaded) {
655+
overloadReason = self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold
656+
? @"consecutive-frames-over-budget"
657+
: @"average-latency-over-budget";
658+
} else if (strained) {
659+
overloadReason = averageLoadPercent >= XCWEncoderStrainedLoadPercent
660+
? @"average-latency-near-budget"
661+
: @"consecutive-frames-near-budget";
662+
}
624663
stats = @{
625664
@"inputFrames": @(inputFrameCount),
626665
@"pendingReplacements": @(pendingReplacementCount),
@@ -631,6 +670,18 @@ - (NSDictionary *)statsRepresentation {
631670
@"inFlightFrames": @(self->_inFlightFrameCount),
632671
@"maxInFlightFrames": @(self->_maxInFlightFrameCount),
633672
@"latestEncodeLatencyUs": @(self->_latestEncodeLatencyUs),
673+
@"averageEncodeLatencyUs": @(self->_averageEncodeLatencyUs),
674+
@"peakEncodeLatencyUs": @(self->_peakEncodeLatencyUs),
675+
@"encoderBudgetUs": @(encoderBudgetUs),
676+
@"encoderLoadPercent": @(latestLoadPercent),
677+
@"averageEncoderLoadPercent": @(averageLoadPercent),
678+
@"overloadState": overloadState,
679+
@"overloaded": @(overloaded),
680+
@"overloadReason": overloadReason,
681+
@"overBudgetFrames": @(self->_overBudgetFrameCount),
682+
@"consecutiveOverBudgetFrames": @(self->_consecutiveOverBudgetFrameCount),
683+
@"consecutiveStrainedFrames": @(self->_consecutiveStrainedFrameCount),
684+
@"overloadEvents": @(self->_overloadEventCount),
634685
@"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs),
635686
@"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0),
636687
@"softwarePacedFrames": @(self->_softwarePacedFrameCount),
@@ -727,6 +778,17 @@ - (uint64_t)maximumHardwareFrameIntervalUsLocked {
727778
return XCWLocalStreamMaximumFrameIntervalUs();
728779
}
729780

781+
- (uint64_t)activeFrameIntervalUsLocked {
782+
if (_encoderMode == XCWVideoEncoderModeH264Software) {
783+
return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked];
784+
}
785+
if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) {
786+
return _hardwareFrameIntervalUs > 0 ? _hardwareFrameIntervalUs : [self initialHardwareFrameIntervalUsLocked];
787+
}
788+
int32_t expectedFrameRate = MAX(1, [self expectedFrameRateLocked]);
789+
return (uint64_t)llround(1000000.0 / (double)expectedFrameRate);
790+
}
791+
730792
- (int32_t)expectedFrameRateLocked {
731793
if (_encoderMode == XCWVideoEncoderModeH264Software) {
732794
if (_lowLatencyMode) {
@@ -1075,6 +1137,12 @@ - (void)invalidateCompressionSessionLocked {
10751137
_inFlightFrameCount = 0;
10761138
_lastSoftwareSubmissionUs = 0;
10771139
_lastHardwareSubmissionUs = 0;
1140+
_latestEncodeLatencyUs = 0;
1141+
_averageEncodeLatencyUs = 0;
1142+
_peakEncodeLatencyUs = 0;
1143+
_consecutiveOverBudgetFrameCount = 0;
1144+
_consecutiveStrainedFrameCount = 0;
1145+
_wasOverloaded = NO;
10781146
_hardwareAccelerated = NO;
10791147
_selectedEncoderID = nil;
10801148
_scalingActive = NO;
@@ -1303,6 +1371,34 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer
13031371
if (submittedAtUs > 0) {
13041372
uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
13051373
_latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0;
1374+
_peakEncodeLatencyUs = MAX(_peakEncodeLatencyUs, _latestEncodeLatencyUs);
1375+
_averageEncodeLatencyUs = _averageEncodeLatencyUs <= 0.0
1376+
? (double)_latestEncodeLatencyUs
1377+
: (_averageEncodeLatencyUs * (1.0 - XCWEncoderLatencyEWMAAlpha)) + ((double)_latestEncodeLatencyUs * XCWEncoderLatencyEWMAAlpha);
1378+
uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked];
1379+
double averageLoadPercent = encoderBudgetUs > 0
1380+
? (_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
1381+
: 0.0;
1382+
double latestLoadPercent = encoderBudgetUs > 0
1383+
? ((double)_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
1384+
: 0.0;
1385+
if (encoderBudgetUs > 0 && _latestEncodeLatencyUs > encoderBudgetUs) {
1386+
_overBudgetFrameCount += 1;
1387+
_consecutiveOverBudgetFrameCount += 1;
1388+
} else {
1389+
_consecutiveOverBudgetFrameCount = 0;
1390+
}
1391+
if (latestLoadPercent >= XCWEncoderStrainedLoadPercent) {
1392+
_consecutiveStrainedFrameCount += 1;
1393+
} else {
1394+
_consecutiveStrainedFrameCount = 0;
1395+
}
1396+
BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent ||
1397+
_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
1398+
if (overloaded && !_wasOverloaded) {
1399+
_overloadEventCount += 1;
1400+
}
1401+
_wasOverloaded = overloaded;
13061402
[self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs];
13071403
[self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
13081404
}

‎client/src/api/types.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,26 @@
1+
export interface EncoderStats {
2+
averageEncodeLatencyUs?: number;
3+
averageEncoderLoadPercent?: number;
4+
consecutiveOverBudgetFrames?: number;
5+
encoderBudgetUs?: number;
6+
encoderLoadPercent?: number;
7+
encoderMode?: string;
8+
hardwareAccelerated?: boolean;
9+
latestEncodeLatencyUs?: number;
10+
overloadEvents?: number;
11+
overloaded?: boolean;
12+
overloadReason?: string;
13+
overloadState?: "nominal" | "strained" | "overloaded" | string;
14+
peakEncodeLatencyUs?: number;
15+
selectedEncoderId?: string | null;
16+
}
17+
118
export interface PrivateDisplayInfo {
219
displayReady: boolean;
320
displayStatus: string;
421
displayWidth: number;
522
displayHeight: number;
23+
encoder?: EncoderStats;
624
frameSequence: number;
725
rotationQuarterTurns?: number;
826
}

‎client/src/app/AppShell.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1669,6 +1669,7 @@ export function AppShell({
16691669
debugPanel={
16701670
debugVisible ? (
16711671
<DebugPanel
1672+
encoder={selectedSimulator.privateDisplay?.encoder}
16721673
fps={fps}
16731674
inline
16741675
onClose={() => setDebugVisible(false)}
@@ -1827,6 +1828,12 @@ function normalizeStreamQuality(
18271828
if (normalized === "economy" || normalized === "ci-software") {
18281829
return "economy";
18291830
}
1831+
if (normalized === "low") {
1832+
return "low";
1833+
}
1834+
if (normalized === "tiny") {
1835+
return "tiny";
1836+
}
18301837
return fallback;
18311838
}
18321839

‎client/src/features/simulators/SimulatorMenu.tsx‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -237,9 +237,11 @@ const STREAM_QUALITY_OPTIONS: Array<{
237237
label: string;
238238
value: StreamQualityPreset;
239239
}> = [
240-
{ label: "Quality", value: "quality" },
241-
{ label: "Balanced", value: "balanced" },
242-
{ label: "Economy", value: "economy" },
240+
{ label: "Full", value: "quality" },
241+
{ label: "1280", value: "balanced" },
242+
{ label: "1080", value: "economy" },
243+
{ label: "720", value: "low" },
244+
{ label: "540", value: "tiny" },
243245
];
244246

245247
function MenuIcon() {

‎client/src/features/stream/streamTypes.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@ export type StreamQualityPreset =
1414
| "ci-software"
1515
| "economy"
1616
| "fast"
17+
| "low"
1718
| "quality"
18-
| "smooth";
19+
| "smooth"
20+
| "tiny";
1921

2022
export interface StreamConfig {
2123
encoder: StreamEncoder;

‎client/src/features/toolbar/DebugPanel.tsx‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1+
import type { EncoderStats } from "../../api/types";
12
import type {
23
StreamRuntimeInfo,
34
StreamStats,
45
StreamStatus,
56
} from "../stream/streamTypes";
67

78
interface DebugPanelProps {
9+
encoder?: EncoderStats | null;
810
fps: number;
911
inline?: boolean;
1012
onClose?: () => void;
@@ -27,6 +29,20 @@ function formatMs(value: number): string {
2729
return `${value.toFixed(1)} ms`;
2830
}
2931

32+
function formatUsAsMs(value: number | undefined): string {
33+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
34+
return "—";
35+
}
36+
return `${(value / 1000).toFixed(1)} ms`;
37+
}
38+
39+
function formatPercent(value: number | undefined): string {
40+
if (typeof value !== "number" || !Number.isFinite(value)) {
41+
return "—";
42+
}
43+
return `${value.toFixed(0)}%`;
44+
}
45+
3046
function formatResolution(stats: StreamStats): string {
3147
if (!stats.width || !stats.height) {
3248
return "—";
@@ -35,6 +51,7 @@ function formatResolution(stats: StreamStats): string {
3551
}
3652

3753
export function DebugPanel({
54+
encoder,
3855
fps,
3956
inline = false,
4057
onClose,
@@ -60,6 +77,26 @@ export function DebugPanel({
6077
{ label: "Frame Gap", value: formatMs(stats.latestFrameGapMs) },
6178
{ label: "Path", value: runtimeInfo.streamBackend },
6279
];
80+
if (encoder) {
81+
rows.push(
82+
{ label: "Encoder", value: encoder.encoderMode ?? "—" },
83+
{ label: "Encoder State", value: encoder.overloadState ?? "—" },
84+
{
85+
label: "Encoder Load",
86+
value: formatPercent(encoder.averageEncoderLoadPercent),
87+
},
88+
{
89+
label: "Encode Latency",
90+
value: formatUsAsMs(encoder.averageEncodeLatencyUs),
91+
},
92+
{ label: "Encode Budget", value: formatUsAsMs(encoder.encoderBudgetUs) },
93+
{ label: "Encoder Reason", value: encoder.overloadReason ?? "—" },
94+
{
95+
label: "Overload Events",
96+
value: String(encoder.overloadEvents ?? 0),
97+
},
98+
);
99+
}
63100

64101
return (
65102
<section

‎docs/api/health.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,20 @@ Returns a snapshot of every server-side counter and the rolling buffer of client
5252
"avg_send_queue_depth": 0.91,
5353
"max_send_queue_depth": 2,
5454
"latest_first_frame_ms": 412,
55+
"encoders": [
56+
{
57+
"udid": "9D7E5BB7-...",
58+
"encoder": {
59+
"encoderMode": "hardware",
60+
"hardwareAccelerated": true,
61+
"overloadState": "nominal",
62+
"averageEncoderLoadPercent": 42.1,
63+
"averageEncodeLatencyUs": 7021.0,
64+
"encoderBudgetUs": 16667,
65+
"overloadEvents": 0
66+
}
67+
}
68+
],
5569
"client_streams": [
5670
{
5771
"clientId": "browser-ABC",
@@ -86,6 +100,21 @@ Returns a snapshot of every server-side counter and the rolling buffer of client
86100
| `max_send_queue_depth` | Peak broadcast channel pressure. |
87101
| `latest_first_frame_ms` | First-frame latency for the most recent connect, in milliseconds. |
88102

103+
### Encoder overload state
104+
105+
`encoders` contains one entry per active simulator session. `encoder.overloadState`
106+
is derived from native VideoToolbox submit-to-output latency:
107+
108+
| State | Meaning |
109+
| ------------ | ---------------------------------------------------------------------------------------- |
110+
| `nominal` | Smoothed encode latency is comfortably below the active frame budget. |
111+
| `strained` | Smoothed latency is near the frame budget or several frames are close to budget. |
112+
| `overloaded` | Smoothed latency exceeds the budget or several frames in a row took longer than budget. |
113+
114+
This is an inferred pressure signal rather than a private macOS hardware queue
115+
counter. It is useful for deciding when to lower stream resolution/FPS or switch
116+
from hardware to software encoding.
117+
89118
### Client stream stats
90119

91120
`client_streams` is a rolling buffer of the most recent reports a client posted to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair.

‎docs/api/rest.md‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ to mirror the daemon's WebRTC ICE configuration.
3636

3737
### `GET /api/metrics`
3838

39-
Returns server-side video stats and a rolling buffer of client-side stats. See [Video Pipeline](/guide/video#tuning-with-metrics) for an annotated example.
39+
Returns server-side video stats, active encoder overload states, and a rolling
40+
buffer of client-side stats. See [Video Pipeline](/guide/video#tuning-with-metrics)
41+
for an annotated example.
4042

4143
### `GET /api/client-stream-stats`
4244

@@ -88,10 +90,11 @@ quality.
8890

8991
`videoCodec` accepts `hardware` or `software` from the UI, and the API also
9092
accepts `auto`. `fps` is clamped to the local stream range. Browser viewers show
91-
three profiles: `quality`, `balanced`, and `economy`. The API still accepts the
92-
legacy `fast`, `smooth`, and `ci-software` profiles for CLI/provider
93-
compatibility. When `profile` is provided, its resolution preset is applied;
94-
send `maxEdge` without `profile` for a custom resolution cap.
93+
five profiles: `quality` (4096 px), `balanced` (1280 px), `economy` (1080 px),
94+
`low` (720 px), and `tiny` (540 px). The API still accepts the legacy `fast`,
95+
`smooth`, and `ci-software` profiles for CLI/provider compatibility. When
96+
`profile` is provided, its resolution preset is applied; send `maxEdge` without
97+
`profile` for a custom resolution cap.
9598

9699
## Simulator inventory
97100

‎docs/cli/flags.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas
3636
| `--client-root` | bundled `client/dist` | Override the static browser client directory. |
3737
| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). |
3838
| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. |
39-
| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, or `ci-software`. |
39+
| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. |
4040
| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. |
4141
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
4242

‎docs/guide/daemon.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and
6262
| `--client-root` | bundled `client/dist` | Override the static browser client directory. |
6363
| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). |
6464
| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. |
65-
| `--stream-quality` | `smooth` | Realtime stream quality profile, including `ci-software` for CI providers. |
65+
| `--stream-quality` | `smooth` | Realtime stream quality profile, including `low`, `tiny`, and `ci-software`. |
6666
| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. |
6767
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
6868

0 commit comments

Comments
 (0)