diff --git a/apps/client/native/screen-share/index.d.ts b/apps/client/native/screen-share/index.d.ts index 6abebff9..86c8f4ff 100644 --- a/apps/client/native/screen-share/index.d.ts +++ b/apps/client/native/screen-share/index.d.ts @@ -16,6 +16,8 @@ export type { NativeScreenAudioPreferences } from '@monky/shared'; export type NativeScreenCaptureEncoder = 'auto' | 'h264_texture_amf' | 'obs_nvenc_h264_tex' | 'obs_x264' | 'av1_texture_amf' | 'obs_nvenc_av1_tex' | 'monky_aom_av1' | 'apple_vt_h264' | 'apple_vt_h264_software'; +/** The H.264 profile a capture encodes: Main, or the Constrained Baseline of the compatible format (#858). */ +export type NativeScreenH264Profile = 'main' | 'constrained-baseline'; export type NativeScreenCaptureTarget = | MacCaptureTarget | { platform?: never; kind: 'window' | 'game'; hwnd: number; expectedProcessId: number; expectedProcessCreationTime100ns: string } @@ -68,6 +70,8 @@ export interface NativeScreenCaptureProbeOptions { runDirectory: string; video: { width: number; height: number; fps: number; bitrateKbps: number; scaleMode?: 'stretch' | 'fit' }; encoder?: NativeScreenCaptureEncoder; + /** H.264 encoders only; Main when absent. */ + h264Profile?: NativeScreenH264Profile; } export interface NativeScreenCaptureProbeResult extends NativeScreenCaptureCapability { @@ -189,6 +193,8 @@ export interface NativeScreenEndpointOptions { audio?: NativeScreenAudioOptions; target?: NativeScreenCaptureTarget | LegacyNativeWindowCaptureTarget; captureEncoder?: NativeScreenCaptureEncoder; + /** The H.264 profile an H.264 publisher encodes and negotiates; Main when absent. */ + h264Profile?: NativeScreenH264Profile; preserveAspectRatio?: boolean; captureDirectory?: string; isSourcePaused?: () => boolean; diff --git a/apps/client/native/screen-share/runtime/captureBridge.cjs b/apps/client/native/screen-share/runtime/captureBridge.cjs index d6e3566d..8b4a0581 100644 --- a/apps/client/native/screen-share/runtime/captureBridge.cjs +++ b/apps/client/native/screen-share/runtime/captureBridge.cjs @@ -14,6 +14,9 @@ class CaptureBridge extends ObsHostBridge { assert.equal(path.basename(runDirectory), `monky-screen-capture-${runId}`); const video = Object.freeze(protocol.normalizedVideo(options.video)); const encoder = protocol.validateEncoder(options.encoder ?? 'auto'); + const h264Profile = options.h264Profile ?? 'main'; + assert.ok(h264Profile === 'main' || (h264Profile === 'constrained-baseline' && encoder !== 'auto' + && protocol.ENCODERS[encoder].codec === 'h264'), 'Constrained Baseline needs an explicit H.264 encoder.'); const bitrateCeilingKbps = options.bitrateCeilingKbps ?? 80000; assert.ok(Number.isSafeInteger(bitrateCeilingKbps) && bitrateCeilingKbps >= video.bitrateKbps && bitrateCeilingKbps <= 80000 && bitrateCeilingKbps % 50 === 0, 'Invalid capture bitrate ceiling.'); @@ -22,7 +25,7 @@ class CaptureBridge extends ObsHostBridge { super({ ...options, protocol: { ...protocol, maxLineBytes: protocol.MAX_LINE_BYTES, allowUnpreparedStop: true, - validateMessage: (message, expected) => protocol.validateMessage(message, { ...expected, video, encoder }), + validateMessage: (message, expected) => protocol.validateMessage(message, { ...expected, video, encoder, h264Profile }), errorDetails: message => message.error }, argumentsForSource: (source, id) => [ `--runtime=${runtime.stockDirectory}`, `--run-directory=${runDirectory}`, `--run-id=${id}`, @@ -31,6 +34,7 @@ class CaptureBridge extends ObsHostBridge { `--height=${video.height}`, `--fps=${video.fps}`, `--bitrate=${video.bitrateKbps}`, `--scale-mode=${video.scaleMode}`, ...(encoder === 'obs_nvenc_av1_tex' ? [`--bitrate-ceiling=${bitrateCeilingKbps}`] : []), + ...(h264Profile === 'constrained-baseline' ? ['--h264-profile=constrained-baseline'] : []), ], validateRetirement: bridge => { const terminal = bridge.stopped ?? bridge.failure; diff --git a/apps/client/native/screen-share/runtime/captureProtocol.cjs b/apps/client/native/screen-share/runtime/captureProtocol.cjs index 3e924f7f..4d016cef 100644 --- a/apps/client/native/screen-share/runtime/captureProtocol.cjs +++ b/apps/client/native/screen-share/runtime/captureProtocol.cjs @@ -98,11 +98,18 @@ function normalizedVideo(video) { return { ...video, scaleMode: video.scaleMode ?? 'stretch' }; } -function configuration(video, encoder = 'h264_texture_amf', kind = 'window') { +/** The H.264 profile a capture encodes (#858); AV1 keeps the legacy `main` label. */ +function validateH264Profile(profile, encoder) { + assert.ok(profile === 'main' || profile === 'constrained-baseline', 'Unknown H.264 capture profile.'); + assert.ok(profile === 'main' || ENCODERS[encoder]?.codec === 'h264', 'Only H.264 encoders have a Constrained Baseline.'); + return profile; +} + +function configuration(video, encoder = 'h264_texture_amf', kind = 'window', h264Profile = 'main') { validateVideo(video); assert.ok(typeof encoder === 'string' && Object.hasOwn(ENCODERS, encoder)); assert.ok(['window', 'monitor', 'game'].includes(kind)); - return { ...CONFIGURATION, codec: ENCODERS[encoder].codec, scaleMode: video.scaleMode ?? 'stretch', encoderId: encoder, rateControl: ENCODERS[encoder].rateControl, + return { ...CONFIGURATION, profile: validateH264Profile(h264Profile, encoder), codec: ENCODERS[encoder].codec, scaleMode: video.scaleMode ?? 'stretch', encoderId: encoder, rateControl: ENCODERS[encoder].rateControl, method: kind === 'game' ? 'game-hook' : 'wgc', width: video.width, height: video.height, fpsNumerator: video.fps, initialBitrateKbps: video.bitrateKbps }; } @@ -226,13 +233,14 @@ function validateMessage(message, expected) { } const encoder = validateEncoder(expected.encoder ?? 'auto'); if (encoder !== 'auto') assert.equal(message.configuration.encoderId, encoder); - assert.deepEqual(message.configuration, configuration(expected.video, message.configuration.encoderId, kind)); + assert.deepEqual(message.configuration, configuration(expected.video, message.configuration.encoderId, kind, + expected.h264Profile ?? 'main')); } const selected = message.configuration; assert.deepEqual(selected, configuration({ width: selected?.width, height: selected?.height, fps: selected?.fpsNumerator, bitrateKbps: selected?.initialBitrateKbps, scaleMode: selected?.scaleMode, - }, selected?.encoderId, kind)); + }, selected?.encoderId, kind, selected?.profile)); if (!extended) assert.equal(selected.encoderId, CONFIGURATION.encoderId, 'Legacy protocol cannot claim NVENC support.'); if (extended) { if (message.capability === null) assert.ok(message.type === 'error' || message.type === 'stopped'); diff --git a/apps/client/native/screen-share/runtime/mac/capture.cjs b/apps/client/native/screen-share/runtime/mac/capture.cjs index a65cb941..4a389372 100644 --- a/apps/client/native/screen-share/runtime/mac/capture.cjs +++ b/apps/client/native/screen-share/runtime/mac/capture.cjs @@ -7,13 +7,14 @@ const { normalizedVideo } = require('../captureProtocol.cjs'); const { notifyObserver } = require('../asyncTools.cjs'); class MacVideoCapture { - constructor({ target, video, mode, codec = 'h264', onPacket, onError, directory }, dependencies = {}) { + constructor({ target, video, mode, codec = 'h264', h264Profile = 'main', onPacket, onError, directory }, dependencies = {}) { assert.ok(mode === 'hardware' || mode === 'software'); assert.ok(codec === 'h264' || codec === 'av1' && mode === 'software'); assert.equal(typeof onPacket, 'function'); assert.equal(typeof onError, 'function'); this.target = validateMacTarget(target); - this.video = Object.freeze({ ...normalizedVideo(video), mode, codec }); + assert.ok(h264Profile === 'main' || (h264Profile === 'constrained-baseline' && codec === 'h264')); + this.video = Object.freeze({ ...normalizedVideo(video), mode, codec, ...(h264Profile === 'main' ? {} : { h264Profile }) }); this.errors = []; this.onError = onError; this.firstFrame = new Promise((resolve, reject) => { diff --git a/apps/client/native/screen-share/runtime/mac/captureBridge.cjs b/apps/client/native/screen-share/runtime/mac/captureBridge.cjs index 2592b7bf..e38936a1 100644 --- a/apps/client/native/screen-share/runtime/mac/captureBridge.cjs +++ b/apps/client/native/screen-share/runtime/mac/captureBridge.cjs @@ -15,9 +15,15 @@ function encoderProfile(encoder) { assert.ok(Object.hasOwn(ENCODERS, encoder), 'The macOS backend requires an explicit native encoder.'); return ENCODERS[encoder]; } -async function probe(host, video, encoder, signal) { +/** The video a macOS capture encodes: the compatible format (#858) names its Constrained Baseline. */ +function encodedVideo(video, mode, codec, h264Profile = 'main') { + assert.ok(h264Profile === 'main' || (h264Profile === 'constrained-baseline' && codec === 'h264'), + 'Only H.264 has a Constrained Baseline.'); + return { ...normalizedVideo(video), mode, codec, ...(h264Profile === 'main' ? {} : { h264Profile }) }; +} +async function probe(host, video, encoder, signal, h264Profile = 'main') { const { mode, codec } = encoderProfile(encoder); - const { value } = await host.request('media.probe', { video: { ...normalizedVideo(video), mode, codec } }, signal); + const { value } = await host.request('media.probe', { video: encodedVideo(video, mode, codec, h264Profile) }, signal); assert.equal(value.captureStarted, false); assert.equal(value.nativeClosed, true); assert.equal(value.hardwareExecutionObserved, null); @@ -27,7 +33,10 @@ async function probe(host, video, encoder, signal) { if (codec === 'av1') { assert.equal(value.profileLevelId, null); assert.equal(value.sequenceVerified, true); - } else assert.match(value.profileLevelId, /^4d[0-9a-f]{4}$/u); + } else if (h264Profile === 'main') assert.match(value.profileLevelId, /^4d[0-9a-f]{4}$/u); + // Constrained Baseline: profile 66 with constraint_set1 (H.264 A.2.1). + else assert.ok(/^42[0-9a-f]{4}$/u.test(value.profileLevelId) && (Number.parseInt(value.profileLevelId.slice(2, 4), 16) & 0x40), + 'The macOS encoder did not produce Constrained Baseline.'); return Object.freeze({ encoderId: encoder, codec, mode, probe: codec === 'av1' ? 'libaom-session' : 'videotoolbox-session', probeVerified: true, textureInput: codec === 'h264', dynamicBitrate: true, hardwareSessionConfirmed: false, hardwareQualified: false }); @@ -59,6 +68,8 @@ class MacCaptureBridge { this.dependencies = dependencies; this.mode = encoderProfile(options.encoder).mode; this.codec = encoderProfile(options.encoder).codec; + this.h264Profile = options.h264Profile ?? 'main'; + encodedVideo(options.video, this.mode, this.codec, this.h264Profile); this.video = Object.freeze(normalizedVideo(options.video)); this.ceiling = options.bitrateCeilingKbps ?? 80000; this.sequence = 0; @@ -69,11 +80,11 @@ class MacCaptureBridge { assert.ok(!this.#capture && !this.stopping, 'A retired or prepared macOS capture owner cannot be reused.'); this.target = Object.freeze(structuredClone(validateMacTarget(target))); this.#capture = new MacVideoCapture({ target: this.target, video: this.video, mode: this.mode, codec: this.codec, - onPacket: this.options.onPacket, onError: this.options.onError }, + h264Profile: this.h264Profile, onPacket: this.options.onPacket, onError: this.options.onError }, { ...this.dependencies, runtime: this.options.host }); const { value } = await this.#capture.host.request('resolve', { target: this.target }, signal); assert.deepEqual(value.target, this.target); - this.capability = await probe(this.#capture.host, this.video, this.options.encoder, signal); + this.capability = await probe(this.#capture.host, this.video, this.options.encoder, signal, this.h264Profile); return this.capability; } async start(target, signal) { @@ -125,4 +136,4 @@ class MacCaptureBridge { } const assertMacCaptureBridgeClosed = Function.prototype.call.bind(MacCaptureBridge.prototype.assertClosed); -module.exports = { MacCaptureBridge, probeCaptureCapabilities, encoderProfile, assertMacCaptureBridgeClosed }; +module.exports = { MacCaptureBridge, probeCaptureCapabilities, encoderProfile, encodedVideo, assertMacCaptureBridgeClosed }; diff --git a/apps/client/native/screen-share/runtime/nativeEndpoint.cjs b/apps/client/native/screen-share/runtime/nativeEndpoint.cjs index 27438ab3..4c77957c 100644 --- a/apps/client/native/screen-share/runtime/nativeEndpoint.cjs +++ b/apps/client/native/screen-share/runtime/nativeEndpoint.cjs @@ -70,6 +70,10 @@ class NativeScreenEndpoint { assert.equal(captureEncoderProfile(options.captureEncoder ?? 'auto', target).codec, source.codec ?? 'h264', 'The selected capture encoder must match the announced screen codec.'); if (options.preserveAspectRatio !== undefined) assert.equal(typeof options.preserveAspectRatio, 'boolean'); + if (options.h264Profile !== undefined) { + assert.ok(['main', 'constrained-baseline'].includes(options.h264Profile), 'Unknown H.264 screen profile.'); + assert.notEqual(source.codec, 'av1', 'An AV1 screen has no H.264 profile.'); + } assert.ok(path.isAbsolute(captureDirectory)); } else assert.ok(destination && typeof destination.frame?.isDestroyed === 'function'); Object.assign(this, { runtime, role, mode, sessionId, publisherSessionId, channelId, pipelineId, source, profile, @@ -81,6 +85,8 @@ class NativeScreenEndpoint { this.audioApplied = true; this.target = target ? cloneSource(target) : null; this.captureEncoder = options.captureEncoder ?? 'auto'; + // The compatible format's Constrained Baseline (#858): its encoder and its engine negotiate only that profile. + this.h264Profile = options.h264Profile ?? 'main'; this.preserveAspectRatio = options.preserveAspectRatio ?? true; this.isSourcePaused = options.isSourcePaused ?? (() => false); this.assertSourceCurrent = options.assertSourceCurrent ?? (() => {}); @@ -113,7 +119,8 @@ class NativeScreenEndpoint { maxResources: 64, maxDecodedFrames: 16, maximumH264Level: role === 'publish' ? Number.parseInt(getScreenH264ProfileLevelId(profile).slice(-2), 16) : 60, requireAudio: source.audio, - videoInput: role === 'publish' ? (source.codec === 'av1' ? 'encoded-av1' : 'encoded-h264') : 'nv12', + videoInput: role !== 'publish' ? 'nv12' : source.codec === 'av1' ? 'encoded-av1' + : options.h264Profile === 'constrained-baseline' ? 'encoded-h264-cb' : 'encoded-h264', }, event => { if (!this.transport) { this.early.push(event); return; } this.track(this.dispatch(event)); @@ -498,6 +505,7 @@ class NativeScreenEndpoint { this.host = createCaptureBridge({ host: this.runtime.host, runtime: this.runtime.obs, runId, runDirectory: this.runDirectory, encoder: this.captureEncoder, + ...(this.source.codec === 'av1' ? {} : { h264Profile: this.h264Profile }), bitrateCeilingKbps: maxBitrateKbps, video: { width, height, fps, bitrateKbps: Math.min(5000, maxBitrateKbps), scaleMode: this.preserveAspectRatio ? 'fit' : 'stretch' }, diff --git a/apps/client/native/screen-share/runtime/nativeRtc/engine/encoded-contract.json b/apps/client/native/screen-share/runtime/nativeRtc/engine/encoded-contract.json index 2336143b..0537de8d 100644 --- a/apps/client/native/screen-share/runtime/nativeRtc/engine/encoded-contract.json +++ b/apps/client/native/screen-share/runtime/nativeRtc/engine/encoded-contract.json @@ -13,7 +13,7 @@ "peer.configureVideoPlayout": "target=peerId; {minimumDelayMs:integer[0,1000]}, only once before any receivers exist. Applies the public SetJitterBufferMinimumDelay API to video receivers; audio is unchanged. Returns {minimumDelayMs,actualDelayMeasured:false}; a request is not an end-to-end latency measurement." }, "node": { - "options": "Revision6 options plus videoInput:'nv12'|'encoded-h264'|'encoded-av1' (default nv12). 'nv12' engines only receive and export decoded NV12; encoded engines publish one source of their own codec. Encoded source and engine modes must match.", + "options": "Revision6 options plus videoInput:'nv12'|'encoded-h264'|'encoded-h264-cb'|'encoded-av1' (default nv12; encoded-h264-cb is the compatible format's Constrained Baseline). 'nv12' engines only receive and export decoded NV12; encoded engines publish one source of their own codec. Encoded source and engine modes must match.", "submitEncodedFrame": "engine.submitEncodedFrame(sourceId,{frameId,data:Buffer,keyframe,timestampUs,durationUs,ntpTimeMs,pts,dts,timebaseNumerator,timebaseDenominator}) -> {sourceId,frameId,inputBytes,copied:true,networkDeliveryConfirmed:false}; synchronous bounded native copy, not a JS-memory loan, retirement Promise or network delivery proof. PTS/DTS are original canonical signed int64 decimal strings." }, "events": { diff --git a/apps/client/native/screen-share/src/capture/contract.h b/apps/client/native/screen-share/src/capture/contract.h index bf82232f..bf24d02e 100644 --- a/apps/client/native/screen-share/src/capture/contract.h +++ b/apps/client/native/screen-share/src/capture/contract.h @@ -58,7 +58,12 @@ inline const char* ScaleModeName(ScaleMode mode) { return mode == ScaleMode::Fit struct VideoConfiguration { std::uint32_t width = 1920, height = 1080, fps = 120, bitrateKbps = 5000; ScaleMode scaleMode = ScaleMode::Stretch; + // The compatible format (#858): H.264 Constrained Baseline instead of Main. + bool constrainedBaseline = false; }; +inline const char* H264ProfileName(const VideoConfiguration& video) { + return video.constrainedBaseline ? "constrained-baseline" : "main"; +} inline constexpr std::uint32_t kMaximumBitrateKbps = 80000; @@ -281,7 +286,7 @@ inline void ValidatePathEvidence(const PathEvidence& value) { } inline Arguments ParseArguments(std::span input) { - Require(input.size() >= 9 && input.size() <= 17, "Expected capture identity and explicit video configuration", + Require(input.size() >= 9 && input.size() <= 18, "Expected capture identity and explicit video configuration", "ERR_SCREEN_CAPTURE_ARGUMENT"); Arguments value; unsigned seen = 0; @@ -354,6 +359,10 @@ inline Arguments ParseArguments(std::span input) { } else if (name == L"--bitrate-ceiling") { bit = 1048576; value.bitrateCeilingKbps = static_cast(Decimal(text, kMaximumBitrateKbps)); + } else if (name == L"--h264-profile") { + bit = 2097152; + Require(text == L"main" || text == L"constrained-baseline", "Unsupported H264 profile", "ERR_SCREEN_CAPTURE_ARGUMENT"); + value.video.constrainedBaseline = text == L"constrained-baseline"; } else throw ContractError("ERR_SCREEN_CAPTURE_ARGUMENT", "Unknown native host option"); } Require((seen & bit) == 0, "Duplicate native host option", "ERR_SCREEN_CAPTURE_ARGUMENT"); @@ -362,7 +371,9 @@ inline Arguments ParseArguments(std::span input) { ValidateVideoConfiguration(value.video); Require(value.bitrateCeilingKbps >= value.video.bitrateKbps && value.bitrateCeilingKbps % 50 == 0, "Invalid capture bitrate ceiling", "ERR_SCREEN_CAPTURE_VIDEO"); - const auto targetOptions = seen & ~(524288u | 1048576u); + Require(!value.video.constrainedBaseline || (value.encoder != EncoderKind::Auto && !IsAv1(value.encoder)), + "Constrained Baseline needs an explicit H264 encoder", "ERR_SCREEN_CAPTURE_ARGUMENT"); + const auto targetOptions = seen & ~(524288u | 1048576u | 2097152u); if (value.encoderProbe) { Require(targetOptions == ((511u & ~24u) | 2048u | 262144u), "Encoder probing must not select a window, monitor or game", "ERR_SCREEN_CAPTURE_ARGUMENT"); @@ -754,7 +765,10 @@ class BitReader { }; inline void ReadSpsPicture(BitReader& bits, const VideoConfiguration& video) { - Require(bits.Read(8) == 77, "Encoded SPS did not accept H264 main profile", "ERR_SCREEN_CAPTURE_H264"); + // A Baseline SPS is marked Constrained Baseline by NormalizeConstrainedBaseline before it leaves the host. + Require(bits.Read(8) == (video.constrainedBaseline ? 66u : 77u), video.constrainedBaseline + ? "Encoded SPS did not accept H264 constrained baseline profile" : "Encoded SPS did not accept H264 main profile", + "ERR_SCREEN_CAPTURE_H264"); Require((bits.Read(8) & 3u) == 0, "Invalid SPS reserved constraint bits", "ERR_SCREEN_CAPTURE_H264"); const auto level = bits.Read(8); Require(level >= RequiredCaptureH264Level(video) && @@ -937,6 +951,22 @@ inline AmfColorNormalization NormalizeAmfBt709(std::span byt return result; } +// A Baseline encoder (AMF and NVENC "baseline", x264) emits no slice groups or redundant pictures, which is +// Constrained Baseline (H.264 A.2.1); some leave constraint_set1 clear. The compatible format (#858) declares it, +// so receivers that admit only Constrained Baseline accept the stream. The flag byte follows profile_idc (never +// zero), so no emulation prevention depends on it. Empty means no rewrite. +inline std::vector NormalizeConstrainedBaseline(std::span bytes, + const VideoConfiguration& video) { + std::vector result; + if (!video.constrainedBaseline) return result; + VisitAnnexB(bytes, [&](std::span nal, std::size_t begin, std::size_t) { + if ((nal.front() & 0x1fu) != 7 || nal.size() < 4 || nal[1] != 66 || (nal[2] & 0x40u)) return; + if (result.empty()) result.assign(bytes.begin(), bytes.end()); + result[begin + 2] = static_cast(result[begin + 2] | 0x40u); + }); + return result; +} + inline std::vector CompleteH264Keyframe(std::span accessUnit, std::span parameterSets, const VideoConfiguration& video) { const auto picture = InspectAnnexB(accessUnit, video); @@ -1268,7 +1298,7 @@ inline std::string ConfigurationJson(Method method, const VideoConfiguration& vi ",\"scaleMode\":" + JsonString(ScaleModeName(video.scaleMode)) + ",\"rateControl\":" + JsonString(EncoderRateControl(encoder)) + ",\"codec\":" + JsonString(EncoderCodec(encoder)) + ",\"encoderId\":" + JsonString(EncoderId(encoder)) + - ",\"profile\":\"main\",\"bFrames\":0,\"keyframeIntervalSeconds\":1}"; + ",\"profile\":" + JsonString(H264ProfileName(video)) + ",\"bFrames\":0,\"keyframeIntervalSeconds\":1}"; } inline std::string CommonJson(const Common& common, std::string_view type, std::uint64_t sequence, diff --git a/apps/client/native/screen-share/src/capture/contractTest.cpp b/apps/client/native/screen-share/src/capture/contractTest.cpp index 28c97f22..c4eda0fb 100644 --- a/apps/client/native/screen-share/src/capture/contractTest.cpp +++ b/apps/client/native/screen-share/src/capture/contractTest.cpp @@ -22,6 +22,7 @@ struct TestSpsOptions { bool vui = false, videoSignal = true, colourDescription = true, richVui = false; std::uint8_t fullRange = 0, primaries = 0, transfer = 1, matrix = 1; std::uint32_t id = 0, order = 0; + std::uint8_t profile = 77, constraints = 0; }; static std::vector ParameterSets(const VideoConfiguration& video, TestSpsOptions options = {}) { @@ -35,7 +36,7 @@ static std::vector ParameterSets(const VideoConfiguration& video, for (auto copy = value; copy; copy >>= 1) ++width; number(0, width - 1); number(value, width); }; - number(77, 8); number(0, 8); number((std::max)(51u, RequiredCaptureH264Level(video)), 8); + number(options.profile, 8); number(options.constraints, 8); number((std::max)(51u, RequiredCaptureH264Level(video)), 8); ue(options.id); ue(0); ue(options.order); if (options.order == 0) ue(0); else if (options.order == 1) { number(0, 1); ue(0); ue(1); ue(2); ue(0); ue(1); } @@ -506,6 +507,27 @@ int main(int argc, char** argv) { } rejects([&] { RequiredCaptureAv1Level({3840, 2160, 60, 5000}, 4950); }); rejects([&] { RequiredCaptureAv1Level({3840, 2160, 60, 5000}, 80050); }); + { + // A Baseline SPS is declared Constrained Baseline in the compatible format, and only there (#858). + VideoConfiguration baseline{1920, 1080, 60, 5000}; + baseline.constrainedBaseline = true; + VideoConfiguration mainVideo = baseline; + mainVideo.constrainedBaseline = false; + TestSpsOptions plain; + plain.profile = 66; + const auto plainBaseline = ParameterSets(baseline, plain); + const auto marked = NormalizeConstrainedBaseline(plainBaseline, baseline); + check(marked.size() == plainBaseline.size() && plainBaseline[5] == 0 && marked[5] == 0x40 && + std::equal(marked.begin() + 6, marked.end(), plainBaseline.begin() + 6)); + check(InspectAnnexB(marked, baseline).sps && InspectAnnexB(marked, baseline).pps); + TestSpsOptions flagged = plain; + flagged.constraints = 0xc0; + check(NormalizeConstrainedBaseline(ParameterSets(baseline, flagged), baseline).empty()); + check(NormalizeConstrainedBaseline(plainBaseline, mainVideo).empty()); + check(NormalizeConstrainedBaseline(ParameterSets(mainVideo), mainVideo).empty()); + rejects([&] { InspectAnnexB(plainBaseline, mainVideo); }); + rejects([&] { InspectAnnexB(ParameterSets(mainVideo), baseline); }); + } check(RequiredCaptureH264Level({3840, 2160, 120, 20000}) == 60); check(RequiredCaptureH264Level({3840, 2160, 60, 20000}) == 52); check(RequiredCaptureH264Level({1920, 1080, 120, 20000}) == 51); @@ -874,6 +896,31 @@ int main(int argc, char** argv) { game.encoder == EncoderKind::Nvenc && game.expectedCreation == 123456789); gameArguments.back() = L"--encoder=x264"; rejects([&] { ParseArguments(gameArguments); }); + { + // The compatible format (#858): Constrained Baseline, only with an explicit H.264 encoder. + auto baseline = windowArguments; + baseline.insert(baseline.end(), {L"--encoder=obs_x264", L"--h264-profile=constrained-baseline"}); + const auto parsed = ParseArguments(baseline); + check(parsed.video.constrainedBaseline && parsed.encoder == EncoderKind::X264); + check(ConfigurationJson(Method::Wgc, parsed.video, EncoderKind::X264).find("\"profile\":\"constrained-baseline\"") + != std::string::npos); + check(ConfigurationJson(Method::Wgc, ParseArguments(windowArguments).video, EncoderKind::X264) + .find("\"profile\":\"main\"") != std::string::npos); + baseline.back() = L"--h264-profile=main"; + check(!ParseArguments(baseline).video.constrainedBaseline); + for (const auto profile : {L"--h264-profile=high", L"--h264-profile=", L"--h264-profile=Main"}) { + baseline.back() = profile; + rejects([&] { ParseArguments(baseline); }); + } + baseline.back() = L"--h264-profile=constrained-baseline"; + for (const auto encoder : {L"--encoder=auto", L"--encoder=monky_aom_av1", L"--encoder=av1_texture_amf"}) { + baseline[baseline.size() - 2] = encoder; + rejects([&] { ParseArguments(baseline); }); + } + auto duplicated = windowArguments; + duplicated.insert(duplicated.end(), {L"--encoder=obs_x264", L"--h264-profile=main", L"--h264-profile=main"}); + rejects([&] { ParseArguments(duplicated); }); + } gameArguments.pop_back(); rejects([&] { ParseArguments(gameArguments); }); std::vector monitorArguments{ diff --git a/apps/client/native/screen-share/src/capture/host.cpp b/apps/client/native/screen-share/src/capture/host.cpp index 45041f96..650d28c9 100644 --- a/apps/client/native/screen-share/src/capture/host.cpp +++ b/apps/client/native/screen-share/src/capture/host.cpp @@ -260,9 +260,15 @@ class Host { delivered.data = normalizedPacket.bytes.data(); delivered.size = normalizedPacket.bytes.size(); } + auto baselinePacket = NormalizeConstrainedBaseline({delivered.data, delivered.size}, host.arguments_.video); + if (!baselinePacket.empty()) { + delivered.data = baselinePacket.data(); + delivered.size = baselinePacket.size(); + } std::vector independentKeyframe; std::vector av1Sequence; AmfColorNormalization normalizedParameters; + std::vector baselineParameters; std::span parameterSets; if (packet->keyframe) { std::uint8_t* extra = nullptr; @@ -274,6 +280,8 @@ class Host { normalizedParameters = NormalizeAmfBt709(parameterSets, host.arguments_.video, host.capability_.encoder, host.amfColorNormalizationAllowed_); if (!normalizedParameters.bytes.empty()) parameterSets = normalizedParameters.bytes; + baselineParameters = NormalizeConstrainedBaseline(parameterSets, host.arguments_.video); + if (!baselineParameters.empty()) parameterSets = baselineParameters; // Preview may have consumed the first headers before a viewer joins. if (IsAv1(host.capability_.encoder)) { auto keyframe = CompleteAv1Keyframe({delivered.data, delivered.size}, parameterSets, @@ -693,6 +701,11 @@ class Host { ",\"settingsAccepted\":true,\"hardwareApplicationConfirmed\":false,\"fpsApplied\":null}"); } } + // OBS lists the same "baseline" for x264, NVENC and AMF; NormalizeConstrainedBaseline declares it Constrained. + // AV1 encoders keep their pinned "main" profile. + std::string H264EncoderProfile() const { + return arguments_.video.constrainedBaseline && !IsAv1(capability_.encoder) ? "baseline" : "main"; + } void EncoderSettings() { const auto* codec = api().obs_get_encoder_codec(SelectedEncoder()); Require(codec || IsSoftware(capability_.encoder), "The selected hardware codec is not registered for this GPU", @@ -711,7 +724,7 @@ class Host { SetInteger(properties.value, encoderSettings_, "keyint_sec", 1); if (IsSoftware(capability_.encoder)) { if (capability_.encoder == EncoderKind::X264) { - SetString(properties.value, encoderSettings_, "profile", "main"); + SetString(properties.value, encoderSettings_, "profile", H264EncoderProfile()); SetString(properties.value, encoderSettings_, "preset", "veryfast"); SetString(properties.value, encoderSettings_, "tune", "zerolatency"); const auto level = (std::max)(51u, RequiredCaptureH264Level(arguments_.video)); @@ -725,7 +738,7 @@ class Host { return; } if (api().obs_properties_get(properties.value, "profile")) - SetString(properties.value, encoderSettings_, "profile", "main"); + SetString(properties.value, encoderSettings_, "profile", H264EncoderProfile()); SetString(properties.value, encoderSettings_, "preset", IsNvenc(capability_.encoder) ? "p4" : "balanced"); if (api().obs_properties_get(properties.value, "bf")) SetInteger(properties.value, encoderSettings_, "bf", 0); else Require((IsNvenc(capability_.encoder) || IsAv1(capability_.encoder)) && api().obs_data_get_int(encoderSettings_, "bf") == 0, @@ -762,7 +775,7 @@ class Host { api().obs_data_get_int(settings.value, "keyint_sec") == 1, "Encoder changed the selected real-time rate settings", "ERR_SCREEN_CAPTURE_ENCODER_INITIALIZATION"); if (capability_.encoder == EncoderKind::X264) - Require(BoundedString(api().obs_data_get_string(settings.value, "profile"), 32) == "main" && + Require(BoundedString(api().obs_data_get_string(settings.value, "profile"), 32) == H264EncoderProfile() && BoundedString(api().obs_data_get_string(settings.value, "preset"), 32) == "veryfast" && BoundedString(api().obs_data_get_string(settings.value, "tune"), 32) == "zerolatency", "Software H264 changed its selected latency settings", "ERR_SCREEN_CAPTURE_ENCODER_INITIALIZATION"); @@ -770,7 +783,7 @@ class Host { } const bool nvenc = IsNvenc(capability_.encoder); Require(BoundedString(api().obs_data_get_string(settings.value, "rate_control"), 32) == EncoderRateControl(capability_.encoder) && - BoundedString(api().obs_data_get_string(settings.value, "profile"), 32) == "main" && + BoundedString(api().obs_data_get_string(settings.value, "profile"), 32) == H264EncoderProfile() && BoundedString(api().obs_data_get_string(settings.value, "preset"), 32) == (nvenc ? "p4" : "balanced") && BoundedString(api().obs_data_get_string(settings.value, nvenc ? "opts" : "ffmpeg_opts"), 512) == EncoderProfileOptions(capability_.encoder, arguments_.video, arguments_.bitrateCeilingKbps) && diff --git a/apps/client/native/screen-share/src/mac/host.mm b/apps/client/native/screen-share/src/mac/host.mm index 57d6aa1c..ae19fd53 100644 --- a/apps/client/native/screen-share/src/mac/host.mm +++ b/apps/client/native/screen-share/src/mac/host.mm @@ -159,7 +159,10 @@ void Finish() { --operations; } monky::screen::mac::EncoderOptions VideoOptions(NSDictionary* value) { - Exact(value, @[@"width", @"height", @"fps", @"bitrateKbps", @"mode", @"scaleMode", @"codec"]); + NSArray* keys = @[@"width", @"height", @"fps", @"bitrateKbps", @"mode", @"scaleMode", @"codec"]; + // The compatible format (#858) names its H.264 profile; Main when absent. + Exact(value, [value isKindOfClass:NSDictionary.class] && value[@"h264Profile"] + ? [keys arrayByAddingObject:@"h264Profile"] : keys); monky::screen::mac::EncoderOptions result; result.width = static_cast(Number(value[@"width"], 4, 3840)); result.height = static_cast(Number(value[@"height"], 2, 2160)); @@ -171,6 +174,11 @@ void Finish() { Require([value[@"codec"] isEqual:@"h264"] || [value[@"codec"] isEqual:@"av1"]); result.av1 = [value[@"codec"] isEqual:@"av1"]; Require(!result.av1 || !result.hardware, "ERR_MAC_AV1_HARDWARE_UNSUPPORTED"); + if (value[@"h264Profile"]) { + Require([value[@"h264Profile"] isEqual:@"main"] || [value[@"h264Profile"] isEqual:@"constrained-baseline"]); + result.constrained_baseline = [value[@"h264Profile"] isEqual:@"constrained-baseline"]; + Require(!result.av1 || !result.constrained_baseline); + } return result; } uint64_t CaptureId(NSDictionary* arguments) { diff --git a/apps/client/native/screen-share/src/mac/videoEncoder.h b/apps/client/native/screen-share/src/mac/videoEncoder.h index c286458e..db6a4349 100644 --- a/apps/client/native/screen-share/src/mac/videoEncoder.h +++ b/apps/client/native/screen-share/src/mac/videoEncoder.h @@ -27,6 +27,8 @@ struct EncoderOptions { int width = 0, height = 0, fps = 0, bitrate_kbps = 0; bool hardware = false; bool av1 = false; + // The compatible format (#858): H.264 Constrained Baseline instead of Main. + bool constrained_baseline = false; }; class VideoEncoder { public: diff --git a/apps/client/native/screen-share/src/mac/videoEncoder.mm b/apps/client/native/screen-share/src/mac/videoEncoder.mm index 11a05c7d..9ead89e4 100644 --- a/apps/client/native/screen-share/src/mac/videoEncoder.mm +++ b/apps/client/native/screen-share/src/mac/videoEncoder.mm @@ -168,8 +168,10 @@ static void Receive(void* owner, void* context, OSStatus status, VTEncodeInfoFla options.bitrate_kbps * 1000); self.minimum_level = level; self.maximum_bitrate_bps = screen_video::H264LevelMaxBitrate(level); + // VideoToolbox names Constrained Baseline only with an automatic level, never above the one required. self.bitstream = std::make_unique(options.width, options.height, level, - screen_video::H264Profile::Main, level > 52 ? screen_video::H264LevelPolicy::Maximum + options.constrained_baseline ? screen_video::H264Profile::ConstrainedBaseline : screen_video::H264Profile::Main, + level > 52 || options.constrained_baseline ? screen_video::H264LevelPolicy::Maximum : screen_video::H264LevelPolicy::Exact); NSDictionary* specification = options.hardware ? @{(__bridge NSString*)kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: @YES} @@ -194,7 +196,9 @@ static void Receive(void* owner, void* context, OSStatus status, VTEncodeInfoFla self.Integer(kVTCompressionPropertyKey_ExpectedFrameRate, options.fps); self.Integer(kVTCompressionPropertyKey_MaxKeyFrameInterval, options.fps); self.Integer(kVTCompressionPropertyKey_AverageBitRate, options.bitrate_kbps * 1000); - NSString* profile = level > 52 ? (__bridge NSString*)kVTProfileLevel_H264_Main_AutoLevel + // kVTProfileLevel_H264_ConstrainedBaseline_AutoLevel (macOS 12+); older systems answer "not supported". + NSString* profile = options.constrained_baseline ? @"H264_ConstrainedBaseline_AutoLevel" + : level > 52 ? (__bridge NSString*)kVTProfileLevel_H264_Main_AutoLevel : [NSString stringWithFormat:@"H264_Main_%u_%u", static_cast(level / 10), static_cast(level % 10)]; const auto selected = VTSessionSetProperty(self.session, kVTCompressionPropertyKey_ProfileLevel, diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_codec_policy.h b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_codec_policy.h index 8d8cdd51..df15f9fa 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_codec_policy.h +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_codec_policy.h @@ -26,6 +26,8 @@ inline void Require(bool condition, const char* message, MonkyEngineStatus statu struct EncodedConfiguration { std::uint32_t width = 1920, height = 1080, fps = 120; bool av1 = false; + // The compatible format (#858): H.264 Constrained Baseline instead of Main. + bool compatible = false; }; inline std::uint8_t Av1ReceiveLevel(const webrtc::SdpVideoFormat& format) { @@ -76,6 +78,18 @@ inline std::optional ParseEncodedFormat(const webrtc::Sd return parsed; } +// The compatible format (#858) is Constrained Baseline at any level a receiver advertises: browsers advertise +// the default Level 3.1 and decode up to 5.2 (their WebRTC decoder probe decides before they watch, and the +// source never needs more than 5.2), and the SFU forwards without levels. Such a receiver is served as if it +// declared the engine's ceiling. +inline std::optional ParseCompatibleFormat(const webrtc::SdpVideoFormat& format) { + auto negotiated = format; + negotiated.parameters.erase("max-recv-level"); + const auto parsed = policy::ParseFormat(negotiated, kEncodedH264Level); + if (!parsed || parsed->profile != sv::H264Profile::ConstrainedBaseline) return std::nullopt; + return policy::NegotiatedH264{sv::H264Profile::ConstrainedBaseline, kEncodedH264Level}; +} + inline bool AcceptsEncodedFormat(const webrtc::SdpVideoFormat& format, std::uint8_t minimum_level = kEncodedH264Level) { const auto parsed = ParseEncodedFormat(format); @@ -83,9 +97,9 @@ inline bool AcceptsEncodedFormat(const webrtc::SdpVideoFormat& format, } // The level a receiver negotiated with this format, when a source of this codec can send to that receiver at all. -inline std::optional ReceiverLevel(const webrtc::SdpVideoFormat& format, bool av1) { +inline std::optional ReceiverLevel(const webrtc::SdpVideoFormat& format, bool av1, bool compatible = false) { if (av1) return policy::IsScreenAv1Format(format) ? std::optional(Av1ReceiveLevel(format)) : std::nullopt; - const auto parsed = ParseEncodedFormat(format); + const auto parsed = compatible ? ParseCompatibleFormat(format) : ParseEncodedFormat(format); return parsed ? std::optional(parsed->level) : std::nullopt; } diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.cc b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.cc index bc225475..b88241c9 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.cc +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.cc @@ -112,8 +112,8 @@ void ValidateEncodedFrame(const MonkyEngineEncodedFrame& frame) { "Invalid bounded Annex B input metadata"); } -EncodedFactoryBundle CreateEncodedVideoFactory(std::uint8_t maximum_level, bool av1) { - auto context = std::make_shared(maximum_level, av1); +EncodedFactoryBundle CreateEncodedVideoFactory(std::uint8_t maximum_level, bool av1, bool compatible) { + auto context = std::make_shared(maximum_level, av1, compatible); auto field_trials = webrtc::FieldTrials::Create( "WebRTC-Dav1dDecoder-CropToRenderResolution/Enabled/" "WebRTC-FrameDropper/Disabled/" @@ -130,6 +130,7 @@ std::shared_ptr CreateEncodedVideoSource(Host& host, std::uint64_t item.key() == "syncGroup" || item.key() == "enabled", "Unknown encoded source option"); auto configuration = ParseConfiguration(options); configuration.av1 = context->av1; + configuration.compatible = context->compatible; const auto group = SyncGroup(options); const bool enabled = Boolean(options, "enabled"); Require(context != nullptr, "The engine has no external-H264 factory"); diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.h b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.h index d732c5ca..cf4525cc 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.h +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video.h @@ -22,7 +22,7 @@ struct EncodedFactoryBundle { std::unique_ptr field_trials; }; -EncodedFactoryBundle CreateEncodedVideoFactory(std::uint8_t maximum_level, bool av1 = false); +EncodedFactoryBundle CreateEncodedVideoFactory(std::uint8_t maximum_level, bool av1 = false, bool compatible = false); std::shared_ptr CreateEncodedVideoSource( Host& host, std::uint64_t id, const Json& options, const std::shared_ptr& cancellation, diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_checks.h b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_checks.h index 4581c803..75fb8967 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_checks.h +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_checks.h @@ -123,6 +123,31 @@ inline void RunEncodedVideoChecks(const std::function& check(rejected, "Contradictory colour must retain its exact observed values in the error"); check(parser.Sps().colorPrimaries == 2, "Rejected colour must not poison the admitted SPS cache"); } + { + // The compatible format (#858): a Baseline SPS without constraint_set1 leaves declared Constrained Baseline. + std::vector packet{ + 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x33, 0xf4, 0x02, 0x80, 0x2d, 0xd3, 0x50, 0x20, 0x20, 0x20, 0x20, + 0, 0, 0, 1, 0x68, 0xce, 0x3c, 0x80, 0, 0, 0, 1, 0x65, 0xbc, + }; + auto input = frame; + input.data = packet.data(); input.data_bytes = static_cast(packet.size()); + sv::H264Bitstream compatible(1280, 720, kEncodedH264Level, sv::H264Profile::ConstrainedBaseline, + sv::H264LevelPolicy::Maximum); + const auto admitted = Normalize(compatible, input); + auto expected = packet; + expected[6] = 0x40; + check(admitted.hasPicture && admitted.keyFrame && admitted.data == expected && + compatible.Sps().ProfileLevelId() == "424033", + "The compatible format must declare a Baseline stream's Constrained Baseline"); + sv::H264Bitstream main_parser(1280, 720, kEncodedH264Level, sv::H264Profile::Main, sv::H264LevelPolicy::Maximum); + rejects([&] { Normalize(main_parser, input); }); + auto groups = packet; + groups[23] = 0xc4; + input.data = groups.data(); + sv::H264Bitstream sliced(1280, 720, kEncodedH264Level, sv::H264Profile::ConstrainedBaseline, + sv::H264LevelPolicy::Maximum); + rejects([&] { Normalize(sliced, input); }); + } for (const auto bad : {0u, static_cast(kEncodedMaximumPacket + 1)}) { auto copy = frame; copy.data_bytes = bad; rejects([&] { ValidateEncodedFrame(copy); }); } @@ -264,6 +289,43 @@ inline void RunEncodedVideoChecks(const std::function& context->source.reset(); check(factory.GetSupportedFormats() == advertised, "Source retirement must not raise cached capabilities to Main6"); } + { + // The compatible format (#858): Constrained Baseline only, served at any level a receiver advertises. + auto compatible = CreateEncodedVideoFactory(52, false, true); + const auto advertised = compatible.encoder_factory->GetSupportedFormats(); + check(advertised.size() == 1 && policy::ParseFormat(advertised[0], kEncodedH264Level)->profile == + sv::H264Profile::ConstrainedBaseline && advertised[0].parameters.at("profile-level-id") == "42e034", + "The compatible engine must advertise only Constrained Baseline at its ceiling"); + auto firefox = advertised[0]; + firefox.parameters["profile-level-id"] = "42e01f"; + check(compatible.encoder_factory->QueryCodecSupport(firefox, "L1T1").is_supported, + "A browser's default Level 3.1 Constrained Baseline receiver must be served"); + firefox.parameters["max-recv-level"] = "e034"; + check(ReceiverLevel(firefox, false, true) == kEncodedH264Level, + "An extended Constrained Baseline receive level must not refuse the receiver"); + auto main_format = firefox; + main_format.parameters.erase("max-recv-level"); + main_format.parameters["profile-level-id"] = "4d003c"; + check(!compatible.encoder_factory->QueryCodecSupport(main_format, "L1T1").is_supported && + !ReceiverLevel(main_format, false, true), "A compatible source must not negotiate Main"); + auto baseline = main_format; + baseline.parameters["profile-level-id"] = "42001f"; + check(!ReceiverLevel(baseline, false, true), "Plain Baseline is not the compatible format"); + check(!bundle.encoder_factory->QueryCodecSupport(firefox, "L1T1").is_supported, + "A Main engine must still refuse a Constrained Baseline-only receiver"); + Json router{{"codecs", Json::array({Json{{"mimeType", "video/H264"}, + {"parameters", {{"profile-level-id", "42001f"}, {"packetization-mode", 1}}}}, Json{{"mimeType", "video/H264"}, + {"parameters", {{"profile-level-id", "42e01f"}, {"packetization-mode", 1}}}}, Json{{"mimeType", "video/H264"}, + {"parameters", {{"profile-level-id", "4d003c"}, {"packetization-mode", 1}}}}})}}; + const auto accepts = [](const auto& format) { return ReceiverLevel(format, false, true).has_value(); }; + check(peer_detail::SelectScreenSendCodec(router, accepts) == router["codecs"][1], + "The SFU must select the router's Constrained Baseline, whose Level 3.1 is only a default"); + rejects([&] { [[maybe_unused]] EncodedVideoContext invalid(52, true, true); }); + const EncodedConfiguration video{1920, 1080, 60, false, true}; + auto state = std::make_shared(1, "compatible-source", false, [](std::string_view, Json) { return true; }, video); + check(state->Snapshot().at("kind") == "encoded-h264-cb" && state->Snapshot().at("maximumProfileLevelId") == "42e03c", + "The compatible source must describe its Constrained Baseline"); + } rejects([&] { CreateEncodedVideoFactory(50); }); rejects([&] { CreateEncodedVideoFactory(61); }); const auto verify_rate_estimate = [&](double fps, bool valid, std::uint32_t maximum_kbps = 20000) { diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_internal.h b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_internal.h index 2866863d..15fb90b3 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_internal.h +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/encoded_video_internal.h @@ -70,7 +70,8 @@ struct State : std::enable_shared_from_this { explicit State(std::uint64_t source, std::string group, bool initially_enabled, std::function event, EncodedConfiguration configuration = {}) : id(source), sync_group(std::move(group)), video(configuration), enabled(initially_enabled), emit(std::move(event)), - stream(video.width, video.height, MinimumEncodedLevel(video), sv::H264Profile::Main, + stream(video.width, video.height, MinimumEncodedLevel(video), + video.compatible ? sv::H264Profile::ConstrainedBaseline : sv::H264Profile::Main, sv::H264LevelPolicy::Maximum) {} bool Event(std::string_view type, Json data) noexcept { @@ -198,9 +199,9 @@ struct State : std::enable_shared_from_this { void Retired(const Token& token) noexcept; Json Snapshot() const { std::lock_guard lock(mutex); - return {{"kind", video.av1 ? "encoded-av1" : "encoded-h264"}, {"width", video.width}, {"height", video.height}, {"requestedFps", video.fps}, + return {{"kind", video.av1 ? "encoded-av1" : video.compatible ? "encoded-h264-cb" : "encoded-h264"}, {"width", video.width}, {"height", video.height}, {"requestedFps", video.fps}, {"profileLevelId", stream.Verified() ? Json(stream.Sps().ProfileLevelId()) : Json(nullptr)}, - {"maximumProfileLevelId", video.av1 ? Json(nullptr) : Json("4d003c")}, + {"maximumProfileLevelId", video.av1 ? Json(nullptr) : Json(video.compatible ? "42e03c" : "4d003c")}, {"av1SequenceLevel", video.av1 && admitted ? Json(av1_level) : Json(nullptr)}, {"sourceId", id}, {"syncGroup", sync_group}, {"enabled", enabled.load()}, {"paused", paused}, {"closed", closed.load()}, {"failed", failed.load()}, {"admitted", admitted}, {"published", published}, {"released", released}, @@ -362,7 +363,8 @@ class Encoder final : public webrtc::VideoEncoder { initial_bitrate = codec->active ? codec->startBitrate * 1000 : 0; maximum_bitrate = codec->maxBitrate ? codec->maxBitrate * 1000 : kEncodedBitrateCeiling; } else { - const auto setup = policy::MakeEncoderSetup(*codec, settings, {sv::H264Profile::Main, negotiated_level_}, 8); + const auto setup = policy::MakeEncoderSetup(*codec, settings, {state_->video.compatible + ? sv::H264Profile::ConstrainedBaseline : sv::H264Profile::Main, negotiated_level_}, 8); initial_bitrate = setup.initial_bitrate; maximum_bitrate = setup.maximum_bitrate; } @@ -554,9 +556,11 @@ class Encoder final : public webrtc::VideoEncoder { class EncodedVideoContext { public: - explicit EncodedVideoContext(std::uint8_t level, bool is_av1 = false) : maximum_level(level), av1(is_av1) { + explicit EncodedVideoContext(std::uint8_t level, bool is_av1 = false, bool is_compatible = false) + : maximum_level(level), av1(is_av1), compatible(is_compatible) { encoded::Require(level == 51 || level == 52 || level == 60, - "External H264 requires a Main5.1, Main5.2 or Main6 ceiling"); + "External H264 requires a Level 5.1, 5.2 or 6 ceiling"); + encoded::Require(!av1 || !compatible, "The compatible format is H264, not AV1"); } void ValidateSource(const encoded::EncodedConfiguration& video) const { if (!av1 && encoded::MinimumEncodedLevel(video) > maximum_level) @@ -567,6 +571,7 @@ class EncodedVideoContext { bool Admits(std::uint8_t receiver_level) const { return av1 || receiver_level >= maximum_level; } const std::uint8_t maximum_level; const bool av1; + const bool compatible; std::mutex mutex; std::weak_ptr source; }; @@ -580,10 +585,11 @@ class Factory final : public webrtc::VideoEncoderFactory { if (context_->av1) return {policy::ScreenAv1Format()}; // WebRTC caches these formats before a source exists. Never change them after startup. auto formats = policy::SupportedFormats(context_->maximum_level); - std::erase_if(formats, [](const auto& format) { + const auto profile = context_->compatible ? sv::H264Profile::ConstrainedBaseline : sv::H264Profile::Main; + std::erase_if(formats, [profile](const auto& format) { const auto parsed = policy::ParseFormat(format, kEncodedH264Level); Require(parsed.has_value(), "Internal H264 capability is invalid"); - return parsed->profile != sv::H264Profile::Main; + return parsed->profile != profile; }); return formats; } @@ -591,7 +597,7 @@ class Factory final : public webrtc::VideoEncoderFactory { std::optional scalability) const override { std::lock_guard lock(context_->mutex); const auto source = context_->source.lock(); - const auto level = ReceiverLevel(format, context_->av1); + const auto level = ReceiverLevel(format, context_->av1, context_->compatible); return {level && (source ? source->Serves(*level) : context_->Admits(*level)) && (!scalability || *scalability == "L1T1"), false}; } @@ -599,7 +605,7 @@ class Factory final : public webrtc::VideoEncoderFactory { const webrtc::SdpVideoFormat& format) override { std::lock_guard lock(context_->mutex); auto source = context_->source.lock(); - const auto level = ReceiverLevel(format, context_->av1); + const auto level = ReceiverLevel(format, context_->av1, context_->compatible); if (!source || source->stopping.load() || !level || !source->Serves(*level)) { RTC_LOG(LS_ERROR) << "Encoded video requires a live source and a receiver supporting its actual level L1T1"; return nullptr; @@ -633,7 +639,7 @@ class Source final : public VideoSource { const std::string& SyncGroup() const override { return state_->sync_group; } bool Enabled() const override { return state_->enabled.load() && !state_->stopping.load() && !state_->failed.load(); } bool AcceptsSendCodec(const webrtc::SdpVideoFormat& format) const override { - const auto level = ReceiverLevel(format, state_->video.av1); + const auto level = ReceiverLevel(format, state_->video.av1, state_->video.compatible); return level && state_->Serves(*level); } void SetEnabled(bool enabled) override { diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/engine.cc b/apps/client/native/screen-share/src/rtc/inputs/engine/engine.cc index a76adcb0..9043be2e 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/engine.cc +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/engine.cc @@ -141,8 +141,10 @@ struct Frame { class Engine final : public Host, public std::enable_shared_from_this { public: - Engine(MonkyEngineOptions options, MonkyEngineCallbacks callbacks, bool encoded_input = false, bool av1 = false) - : options_(options), encoded_input_(encoded_input), av1_(av1), callbacks_(callbacks), resources_(options.max_resources), + Engine(MonkyEngineOptions options, MonkyEngineCallbacks callbacks, bool encoded_input = false, bool av1 = false, + bool compatible = false) + : options_(options), encoded_input_(encoded_input), av1_(av1), compatible_(compatible), callbacks_(callbacks), + resources_(options.max_resources), receive_routes_(std::size_t(options.max_resources) * receiver_policy::kMaxReceiverHistory) {} ~Engine() override { if (actor_.joinable()) actor_.join(); @@ -553,7 +555,7 @@ class Engine final : public Host, public std::enable_shared_from_this { {"closed", done_}, {"ready", ready_ && !closing_}, {"resources", resources_.Size()}, {"pendingOperations", operations_.size()}, {"pendingServerRequests", replies_.size()}, {"contractRevision", MONKY_ENGINE_CONTRACT_REVISION}, - {"videoInput", encoded_input_ ? (av1_ ? "encoded-av1" : "encoded-h264") : "nv12"}, + {"videoInput", encoded_input_ ? (av1_ ? "encoded-av1" : compatible_ ? "encoded-h264-cb" : "encoded-h264") : "nv12"}, {"activeReceiverRoutes", receive_routes_.Size()}, {"decodedFrames", frames_.size()}, {"droppedFrames", dropped_frames_.load()}, {"rejectedDecodedFrames", received_failures_.Rejected()}, @@ -740,7 +742,7 @@ class Engine final : public Host, public std::enable_shared_from_this { std::unique_ptr video_field_trials = webrtc::FieldTrials::Create("WebRTC-Dav1dDecoder-CropToRenderResolution/Enabled/"); if (encoded_input_) { - auto external = CreateEncodedVideoFactory(options.maximum_h264_level, av1_); + auto external = CreateEncodedVideoFactory(options.maximum_h264_level, av1_, compatible_); encoded_context_ = std::move(external.context); bundle.encoder_factory = std::move(external.encoder_factory); video_field_trials = std::move(external.field_trials); @@ -1231,6 +1233,7 @@ class Engine final : public Host, public std::enable_shared_from_this { const MonkyEngineOptions options_; const bool encoded_input_; const bool av1_; + const bool compatible_; #if defined(_WIN32) CpuFrameUpload cpu_frame_upload_; #endif @@ -1358,7 +1361,7 @@ extern "C" MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_request( } static MonkyEngineStatus CreateEncodedEngine( const MonkyEngineOptions* options, const MonkyEngineCallbacks* callbacks, - MonkyRtcEngine** engine, MonkyEngineError* error, bool av1) noexcept { + MonkyRtcEngine** engine, MonkyEngineError* error, bool av1, bool compatible = false) noexcept { if (engine) *engine = nullptr; return Boundary(error, [&] { if (!engine || !options || !callbacks || callbacks->struct_size != sizeof(*callbacks) || @@ -1370,7 +1373,7 @@ static MonkyEngineStatus CreateEncodedEngine( "External H264 engine requires at least Main Level 5.1; its fixed ceiling must cover the source", MONKY_ENGINE_UNSUPPORTED); auto result = std::make_unique(); - result->state = std::make_shared(*options, *callbacks, true, av1); + result->state = std::make_shared(*options, *callbacks, true, av1, compatible); result->state->Start(); *engine = result.release(); }); @@ -1385,6 +1388,11 @@ extern "C" MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_create_en MonkyRtcEngine** engine, MonkyEngineError* error) noexcept { return CreateEncodedEngine(options, callbacks, engine, error, true); } +extern "C" MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_create_encoded_compatible( + const MonkyEngineOptions* options, const MonkyEngineCallbacks* callbacks, + MonkyRtcEngine** engine, MonkyEngineError* error) noexcept { + return CreateEncodedEngine(options, callbacks, engine, error, false, true); +} extern "C" MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_respond( MonkyRtcEngine* engine, uint64_t id, const char* json, uint32_t bytes, MonkyEngineError* error) noexcept { return Boundary(error, [&] { State(engine).Respond(id, rtc::Parse(json, bytes)); }); diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/monky_rtc_engine.h b/apps/client/native/screen-share/src/rtc/inputs/engine/monky_rtc_engine.h index 7cc34017..77a401e5 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/monky_rtc_engine.h +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/monky_rtc_engine.h @@ -179,6 +179,11 @@ MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_create_encoded_av1( const MonkyEngineOptions* options, const MonkyEngineCallbacks* callbacks, MonkyRtcEngine** engine, MonkyEngineError* error) MONKY_ENGINE_NOEXCEPT; +// External H.264 in Constrained Baseline, the compatible format every browser decodes (#858). +MONKY_ENGINE_API MonkyEngineStatus __cdecl monky_rtc_engine_create_encoded_compatible( + const MonkyEngineOptions* options, const MonkyEngineCallbacks* callbacks, + MonkyRtcEngine** engine, MonkyEngineError* error) MONKY_ENGINE_NOEXCEPT; + // IDs are caller-selected nonzero safe JS integers. Do not reuse a request ID // before its completion event has been consumed. // OK acknowledges admission only. operation events report actual completion. diff --git a/apps/client/native/screen-share/src/rtc/inputs/engine/node/napi_engine.cc b/apps/client/native/screen-share/src/rtc/inputs/engine/node/napi_engine.cc index d2d89928..fa72526e 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/engine/node/napi_engine.cc +++ b/apps/client/native/screen-share/src/rtc/inputs/engine/node/napi_engine.cc @@ -148,7 +148,7 @@ uint32_t Option(napi_env env, napi_value options, const char* name, uint32_t fal "Numeric engine options must be positive uint32 safe integers")); } -MonkyEngineOptions Options(napi_env env, napi_value object, bool& encoded, bool& av1) { +MonkyEngineOptions Options(napi_env env, napi_value object, bool& encoded, bool& av1, bool& compatible) { Record(env, object, "options must be an object"); constexpr std::array names{ "maxResources", "maxPendingOperations", "maxDecodedFrames", @@ -176,9 +176,10 @@ MonkyEngineOptions Options(napi_env env, napi_value object, bool& encoded, bool& options.maximum_h264_level = Option(env, object, "maximumH264Level", 60); if (Has(env, object, "videoInput")) { const auto input = String(env, Get(env, object, "videoInput"), 1, 16, "Invalid videoInput"); - if (input != "nv12" && input != "encoded-h264" && input != "encoded-av1") - Invalid(env, "videoInput must be nv12, encoded-h264 or encoded-av1"); + if (input != "nv12" && input != "encoded-h264" && input != "encoded-h264-cb" && input != "encoded-av1") + Invalid(env, "videoInput must be nv12, encoded-h264, encoded-h264-cb or encoded-av1"); av1 = input == "encoded-av1"; + compatible = input == "encoded-h264-cb"; encoded = input != "nv12"; } if (options.max_resources > 64 || options.max_pending_operations > 128 || @@ -262,8 +263,8 @@ napi_value CreateEngine(napi_env env, napi_callback_info info) { Arguments<2> args(env, info); // Inert DLL negotiation precedes option getters, async state and creation. RequireContract(env); - bool encoded = false, av1 = false; - const auto options = Options(env, args.values[0], encoded, av1); + bool encoded = false, av1 = false, compatible = false; + const auto options = Options(env, args.values[0], encoded, av1, compatible); if (Type(env, args.values[1]) != napi_function) Invalid(env, "onEvent must be a function"); auto state = std::make_shared(env, options); try { @@ -276,6 +277,7 @@ napi_value CreateEngine(napi_env env, napi_callback_info info) { MonkyEngineError error = EmptyError(); MonkyRtcEngine* engine = nullptr; const auto status = av1 ? monky_rtc_engine_create_encoded_av1(&options, &callbacks, &engine, &error) + : compatible ? monky_rtc_engine_create_encoded_compatible(&options, &callbacks, &engine, &error) : encoded ? monky_rtc_engine_create_encoded(&options, &callbacks, &engine, &error) : monky_rtc_engine_create(&options, &callbacks, &engine, &error); { diff --git a/apps/client/native/screen-share/src/rtc/inputs/native_core/h264_bitstream.cc b/apps/client/native/screen-share/src/rtc/inputs/native_core/h264_bitstream.cc index 91b4a57a..4fadd49c 100644 --- a/apps/client/native/screen-share/src/rtc/inputs/native_core/h264_bitstream.cc +++ b/apps/client/native/screen-share/src/rtc/inputs/native_core/h264_bitstream.cc @@ -272,7 +272,14 @@ std::vector> H264Bitstream::Split(std::span nal) { if (nal.size() > maxParameterBytes) throw std::runtime_error("H264 parameter set is too large"); if (Type(nal) == 7) { - auto sps = ParseH264Sps(nal); + std::vector stored(nal.begin(), nal.end()); + // A Baseline stream without slice groups or redundant pictures (checked on its PPS) is Constrained + // Baseline even when the encoder leaves constraint_set1 clear (H.264 A.2.1, #858). The compatible format + // declares it, so receivers that admit only Constrained Baseline accept it. The byte follows profile_idc + // (never zero), so no emulation prevention depends on it. + if (profile_ == H264Profile::ConstrainedBaseline && stored.size() > 3 && stored[1] == 66) + stored[2] = static_cast(stored[2] | 0x40u); + auto sps = ParseH264Sps(stored); const bool levelMatches = levelPolicy_ == H264LevelPolicy::Exact ? sps.levelIdc == level_ : sps.levelIdc <= level_ && H264LevelMaxBitrate(sps.levelIdc) > 0; @@ -281,7 +288,7 @@ void H264Bitstream::Remember(std::span nal) { throw std::runtime_error("SPS profile/level/geometry does not match the requested encoder mode"); } sps_ = sps; - spsById_[sps.id] = {nal.begin(), nal.end()}; + spsById_[sps.id] = std::move(stored); verified_ = true; } else if (Type(nal) == 8) { Bits bits(nal); @@ -291,6 +298,15 @@ void H264Bitstream::Remember(std::span nal) { if (bits.Read(1) && profile_ != H264Profile::Main && profile_ != H264Profile::AnySupported) { throw std::runtime_error("CABAC is not permitted in H264 Baseline"); } + if (profile_ == H264Profile::ConstrainedBaseline) { + bits.Read(1); + if (bits.Ue() != 0) throw std::runtime_error("Slice groups are not permitted in H264 Constrained Baseline"); + bits.Ue(); bits.Ue(); + if (bits.Read(1) || bits.Read(2)) throw std::runtime_error("Weighted prediction is not permitted in H264 Baseline"); + bits.Se(); bits.Se(); bits.Se(); + bits.Read(1); bits.Read(1); + if (bits.Read(1)) throw std::runtime_error("Redundant pictures are not permitted in H264 Constrained Baseline"); + } ppsById_[ppsId] = {nal.begin(), nal.end()}; } std::size_t total = 0; @@ -380,7 +396,10 @@ H264AccessUnit H264Bitstream::Convert(std::span bytes) { } for (const auto nal : nals) { const auto type = Type(nal); - if (type != 9 && !(result.keyFrame && (type == 7 || type == 8))) Append(result.data, nal); + if (type == 9 || (result.keyFrame && (type == 7 || type == 8))) continue; + // An SPS goes out as remembered: the compatible format's declares Constrained Baseline. + if (type == 7) Append(result.data, spsById_.at(ParseH264Sps(nal).id)); + else Append(result.data, nal); } return result; } diff --git a/apps/client/native/screen-share/test/capturePlatforms.test.cjs b/apps/client/native/screen-share/test/capturePlatforms.test.cjs index ed0b7cb9..36776201 100644 --- a/apps/client/native/screen-share/test/capturePlatforms.test.cjs +++ b/apps/client/native/screen-share/test/capturePlatforms.test.cjs @@ -152,6 +152,47 @@ test('capture scaling is explicit, defaults to stretch and cannot silently chang assert.throws(() => protocol.validateVideo({ ...video, scaleMode })); }); +test('the compatible format encodes Constrained Baseline only on an explicit H.264 encoder, and the host cannot relabel it (#858)', async () => { + for (const encoder of ['h264_texture_amf', 'obs_nvenc_h264_tex', 'obs_x264']) { + const baseline = message(windowTarget, encoder); + baseline.configuration = protocol.configuration(video, encoder, 'window', 'constrained-baseline'); + assert.equal(baseline.configuration.profile, 'constrained-baseline'); + const expected = { source: windowTarget, video, runId, helperProcessId: 42, encoder, h264Profile: 'constrained-baseline' }; + protocol.validateMessage(baseline, expected); + // A host that kept Main, or a Main share the host relabels, is refused. + assert.throws(() => protocol.validateMessage(message(windowTarget, encoder), expected)); + assert.throws(() => protocol.validateMessage(baseline, { ...expected, h264Profile: 'main' })); + } + for (const encoder of ['av1_texture_amf', 'obs_nvenc_av1_tex', 'monky_aom_av1']) + assert.throws(() => protocol.configuration(video, encoder, 'window', 'constrained-baseline'), /Constrained Baseline/); + assert.throws(() => protocol.configuration(video, 'obs_x264', 'window', 'high'), /Unknown H.264/); + const options = (encoder, h264Profile) => ({ + host: { kind: 'verified-native-screen-capture-host', executable: path.resolve('host.exe'), sha256: 'a'.repeat(64) }, + runtime: { kind: 'verified-stock-obs-runtime', version: '32.1.1', stockDirectory: path.resolve('obs'), binaryDirectory: path.resolve('obs') }, + runId, runDirectory: path.resolve(`monky-screen-capture-${runId}`), video, encoder, h264Profile, + onError() {}, onPacket() {}, onNotice() {}, + }); + for (const encoder of ['auto', 'monky_aom_av1']) + assert.throws(() => new CaptureBridge(options(encoder, 'constrained-baseline')), /explicit H.264 encoder/); + let args = null; + const child = new EventEmitter(); + child.pid = 42; child.stdout = new PassThrough(); child.stderr = new PassThrough(); + child.stdin = new Writable({ write(_bytes, _encoding, done) { done(); } }); + child.stdio = [child.stdin, child.stdout, child.stderr, new PassThrough(), new Writable({ write(_bytes, _encoding, done) { done(); } })]; + const bridge = new CaptureBridge(options('obs_x264', 'constrained-baseline'), { + spawnProcess(_executable, spawned) { + args = spawned; + const prepared = message(windowTarget, 'obs_x264'); + prepared.configuration = protocol.configuration({ ...video, scaleMode: 'stretch' }, 'obs_x264', 'window', 'constrained-baseline'); + queueMicrotask(() => child.stdout.write(JSON.stringify(prepared) + '\n')); + return child; + }, + }); + await bridge.prepare(windowTarget); + assert.ok(args.includes('--h264-profile=constrained-baseline') && args.includes('--encoder=obs_x264')); + child.emit('exit', 0, null); child.emit('close', 0, null); +}); + test('a renamed window keeps its share while another window class or executable is rejected', () => { for (const target of [windowTarget, gameTarget]) { const prepared = message(target, 'h264_texture_amf'); diff --git a/apps/client/native/screen-share/test/macCaptureBridge.test.cjs b/apps/client/native/screen-share/test/macCaptureBridge.test.cjs index 287340b7..bf43259d 100644 --- a/apps/client/native/screen-share/test/macCaptureBridge.test.cjs +++ b/apps/client/native/screen-share/test/macCaptureBridge.test.cjs @@ -152,3 +152,31 @@ test('macOS retirement proof belongs to the original bridge, not a snapshot, met assert.equal(f.owner.exited, false, 'Releasing a lease must not terminate another owner of the helper.'); await originalClose(); }); + +test('the compatible format asks VideoToolbox for Constrained Baseline and refuses anything else (#858)', async () => { + for (const profileLevelId of ['42e033', '424033']) { + const f = fixture({ probe: { profileLevelId } }); + const bridge = new MacCaptureBridge({ ...f.options, h264Profile: 'constrained-baseline' }, f.dependencies); + await bridge.prepare(target); + await bridge.start(target); + for (const method of ['media.probe', 'media.start']) + assert.equal(f.requests.find(request => request.method === method).data.video.h264Profile, 'constrained-baseline'); + await bridge.stop(); + } + // A Main or plain Baseline stream is not the compatible format. + for (const profileLevelId of ['4d002a', '420033']) { + const f = fixture({ probe: { profileLevelId } }); + const bridge = new MacCaptureBridge({ ...f.options, h264Profile: 'constrained-baseline' }, f.dependencies); + await assert.rejects(bridge.prepare(target), /Constrained Baseline/); + await bridge.stop(); + } + // Main keeps the original request, without a profile field the host would not know. + const main = fixture(); + const bridge = new MacCaptureBridge(main.options, main.dependencies); + await bridge.prepare(target); + assert.equal(Object.hasOwn(main.requests.find(request => request.method === 'media.probe').data.video, 'h264Profile'), false); + await bridge.stop(); + const av1 = fixture(); + assert.throws(() => new MacCaptureBridge({ ...av1.options, encoder: 'monky_aom_av1', h264Profile: 'constrained-baseline' }, + av1.dependencies), /Only H.264/); +}); diff --git a/apps/client/native/screen-share/test/nativeEndpointOwnership.test.cjs b/apps/client/native/screen-share/test/nativeEndpointOwnership.test.cjs index 4d9c30b3..9e242c85 100644 --- a/apps/client/native/screen-share/test/nativeEndpointOwnership.test.cjs +++ b/apps/client/native/screen-share/test/nativeEndpointOwnership.test.cjs @@ -15,7 +15,8 @@ const { decoderFailureObservations } = require('../runtime/nativeVideoDiagnostic function fixture({ incomplete = false, role = 'receive', mode = 'p2p', audio = false, target = { hwnd: 12345, expectedProcessId: 56789 }, captureEncoder = 'auto', captureModule, preserveAspectRatio, video = { width: 1920, height: 1080, fps: 120, maxBitrateKbps: 20000 }, quality = 'source', - captureRuntime = {}, captureDirectory = path.resolve(__dirname, 'modeled-capture'), timers, send = async () => {} } = {}) { + captureRuntime = {}, captureDirectory = path.resolve(__dirname, 'modeled-capture'), timers, send = async () => {}, h264Profile, + codec } = {}) { let finish, closeCalls = 0, engineOptions; const contract = require('../runtime/nativeRtc/engine/contract.json'); const capabilities = { abiVersion: contract.abiVersion, contractRevision: contract.contractRevision, @@ -45,9 +46,9 @@ function fixture({ incomplete = false, role = 'receive', mode = 'p2p', audio = f textures: { importSharedTexture() {}, sendSharedTexture() {} }, role, mode, sessionId: role === 'publish' ? 'publisher' : 'viewer', publisherSessionId: 'publisher', channelId: 'channel', pipelineId: randomUUID(), source: { shareId: 'screen', instanceId: randomUUID(), audio, - video }, + video, ...(codec ? { codec } : {}) }, quality, destination: { frame, presentationId: randomUUID() }, - target, captureEncoder, preserveAspectRatio, captureDirectory, + target, captureEncoder, preserveAspectRatio, captureDirectory, ...(h264Profile ? { h264Profile } : {}), ...(audio ? { audio: { sinkId: '', muted: true, volume: 0, maxBitrateBps: 128000, captureModule: captureModule ?? { createPacketCapture() { assert.fail('Preview-only demand cannot capture PCM.'); } }, output: { webContents, frame, expectedUrl: frame.url, @@ -80,6 +81,26 @@ test('publisher fixes its H264 ceiling before engine startup and source creation } }); +test('the compatible format starts a Constrained Baseline engine, and only for H.264 (#858)', async () => { + for (const mode of ['p2p', 'sfu']) { + const f = fixture({ mode, role: 'publish', h264Profile: 'constrained-baseline', captureEncoder: 'obs_x264' }); + try { + assert.equal(f.engineOptions.videoInput, 'encoded-h264-cb'); + assert.equal(f.endpoint.h264Profile, 'constrained-baseline'); + await f.endpoint.ready; + } finally { + f.finish({ closed: true }); + await f.endpoint.close(); + } + const main = fixture({ mode, role: 'publish', h264Profile: 'main' }); + try { assert.equal(main.engineOptions.videoInput, 'encoded-h264'); } + finally { main.finish({ closed: true }); await main.endpoint.close(); } + } + assert.throws(() => fixture({ role: 'publish', h264Profile: 'high' }), /Unknown H.264/); + assert.throws(() => fixture({ role: 'publish', h264Profile: 'constrained-baseline', codec: 'av1', + captureEncoder: 'monky_aom_av1' }), /no H.264 profile/); +}); + test('a lower-quality rendition of a Main6 source starts its own Main5.1 factory', async () => { for (const mode of ['p2p', 'sfu']) { for (const quality of ['1080p60', '720p60', '480p30']) { diff --git a/apps/client/src/main/nativeScreen/encoderDiscovery.ts b/apps/client/src/main/nativeScreen/encoderDiscovery.ts index b432c89d..89b23042 100644 --- a/apps/client/src/main/nativeScreen/encoderDiscovery.ts +++ b/apps/client/src/main/nativeScreen/encoderDiscovery.ts @@ -7,7 +7,7 @@ import { type ScreenEncodingAvailability, type ScreenEncodingSelection, } from '@monky/shared'; import type { - CaptureBridge, NativeScreenCaptureCapability, NativeScreenCaptureTarget, NativeScreenRuntime, + CaptureBridge, NativeScreenCaptureCapability, NativeScreenCaptureTarget, NativeScreenH264Profile, NativeScreenRuntime, } from '@monky/screen-share'; import { canonicalDirectory, samePath } from '../canonicalDirectory'; import { isRecord } from '../ipc/guards'; @@ -59,7 +59,8 @@ export class SelectedCaptureProbe { } async prepare(target: NativeScreenCaptureTarget, video: NativeScreenVideoProfile, - signal: AbortSignal, encoding: ScreenEncodingSelection, preserveAspectRatio = true): Promise { + signal: AbortSignal, encoding: ScreenEncodingSelection, preserveAspectRatio = true, + h264Profile: NativeScreenH264Profile = 'main'): Promise { let capability: NativeScreenCaptureCapability; try { signal.throwIfAborted(); @@ -71,7 +72,7 @@ export class SelectedCaptureProbe { this.bridge = screenShareModule().createCaptureBridge({ host: this.runtime.host, runtime: this.runtime.obs, runId: this.runId, runDirectory: this.directory, encoder: encoding.encoder, video: { width, height, fps, bitrateKbps: maxBitrateKbps, - scaleMode: preserveAspectRatio ? 'fit' : 'stretch' }, + scaleMode: preserveAspectRatio ? 'fit' : 'stretch' }, ...(encoding.codec === 'h264' ? { h264Profile } : {}), onError: error => { console.warn('[NativeScreen] Selected-source probe failed:', error); this.onError(error); @@ -174,7 +175,7 @@ export class EncoderDiscovery { */ encoding(request: ScreenEncodingRequest, signal?: AbortSignal, refresh = false): Promise { if (this.retirementFailure) return Promise.reject(this.retirementFailure); - const { video, mode, codec, strategy, inspectHardware } = request; + const { video, mode, codec, strategy, inspectHardware, h264Only = false } = request; const abort = signal ? AbortSignal.any([this.abortController.signal, signal]) : this.abortController.signal; const remembered = refresh ? null : this.cache.get(request, this.host.displayGeneration()); if (remembered) return Promise.resolve().then(() => { @@ -243,7 +244,7 @@ export class EncoderDiscovery { encoder: selection.encoder, codec: selection.codec, encodingMode: selection.mode, refusal: outcome, retirementConfirmed: true, continuingHardwareDiscovery: outcome === 'next-candidate', }, outcome === 'failed' ? 'WARN' : 'INFO'); - }); + }, h264Only); this.cache.set(request, displayGeneration, availability, skipped); return availability; }); diff --git a/apps/client/src/main/nativeScreenSharing.ts b/apps/client/src/main/nativeScreenSharing.ts index a0bf8372..cd1dc0c5 100644 --- a/apps/client/src/main/nativeScreenSharing.ts +++ b/apps/client/src/main/nativeScreenSharing.ts @@ -3,7 +3,7 @@ import { randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import { MessageType, NATIVE_SCREEN_IPC, SCREEN_SHARE_LIMITS, nativeScreenCommandSchema, - getScreenShareProfile, nativeScreenFailureReason, nativeScreenProducerSchema, nativeScreenSignalSchema, + compatibleScreenProfile, getScreenShareProfile, nativeScreenFailureReason, nativeScreenProducerSchema, nativeScreenSignalSchema, type NativeScreenCall, type NativeScreenCapabilities, type NativeScreenCommand, type NativeScreenCommandResult, type NativeScreenParticipant, type NativeScreenSignalPayload, type NativeScreenSource, type NativeScreenAudioPreferences, type NativeScreenCaptureKind, type NativeScreenVideoProfile, @@ -162,7 +162,8 @@ class NativeScreenSharingService { if (command.action === 'probe-encoding') { if (this.shutdownRequested || this.disposal) throw cancelled(); return { kind: 'encoding', availability: await this.discovery.verify(command.probeId, { video: command.video, - mode: command.encodingMode, codec: command.codec, strategy: command.encodingStrategy ?? 'automatic', inspectHardware: true }) }; + mode: command.encodingMode, codec: command.codec, strategy: command.encodingStrategy ?? 'automatic', inspectHardware: true, + h264Only: command.screenFormat === 'compatible' }) }; } if (command.action === 'join') { if (this.admissionsFrozen || this.shutdownRequested || this.disposal) throw cancelled(); @@ -427,22 +428,26 @@ class NativeScreenSharingService { assertCurrent(); monitor.assertCurrent(); stage = 'preflight'; - this.diagnostics.log('preflight-start', diagnostic); + // The compatible format is H.264 Constrained Baseline, which Firefox and Safari decode too (#858). + const compatible = command.screenFormat === 'compatible'; + const h264Profile = compatible ? 'constrained-baseline' as const : 'main' as const; + this.diagnostics.log('preflight-start', { ...diagnostic, screenFormat: compatible ? 'compatible' : 'main' }); const discover = (): Promise => this.discovery.encoding({ video: command.video, mode: command.encodingMode ?? 'hardware', codec: command.codec ?? 'auto', - strategy: command.encodingStrategy ?? 'automatic', inspectHardware: false }, selection.abort.signal); + strategy: command.encodingStrategy ?? 'automatic', inspectHardware: false, h264Only: compatible }, selection.abort.signal); type Preflight = { video: NativeScreenVideoProfile; proof: NativeScreenCaptureCapability }; // The selected source proves the chosen encoder with the real target, even when discovery was remembered. const prepare = async (encoding: ScreenEncodingAvailability): Promise => { assertCurrent(); if (!encoding.selection) throw new Error(encoding.reason ?? 'No compatible screen encoder is available for this codec and profile.'); - const video = getScreenShareProfile(command.video, 'source', encoding.selection.codec); + const sized = getScreenShareProfile(command.video, 'source', encoding.selection.codec); + const video = compatible ? compatibleScreenProfile(sized) : sized; diagnostic.video = diagnosticProfile(video); selection.probe = new SelectedCaptureProbe(this.nativeRuntime(), captureDirectory, error => this.diagnostics.logFailure('preflight', error, undefined, { ...diagnostic, stage: 'preflight' })); try { return { video, proof: await selection.probe.prepare(target, video, selection.abort.signal, encoding.selection, - command.preserveAspectRatio) }; + command.preserveAspectRatio, h264Profile) }; } catch (error) { // An encoder that fails with the real source makes every remembered discovery suspect. if (isScreenEncoderFailure(error)) this.discovery.cache.clear(); @@ -543,6 +548,7 @@ class NativeScreenSharingService { source: { ...options.source, codec: source.codec ?? 'h264' }, onState: state => { observe(state); options.onState(state); }, publisherSessionId: call.config.sessionId, target: captureTarget, captureDirectory, captureEncoder: proof.encoderId, + h264Profile, preserveAspectRatio: command.preserveAspectRatio ?? true, isSourcePaused: () => monitor.paused, assertSourceCurrent: () => { diff --git a/apps/client/src/main/screenEncodingCache.ts b/apps/client/src/main/screenEncodingCache.ts index ad335a94..77379f96 100644 --- a/apps/client/src/main/screenEncodingCache.ts +++ b/apps/client/src/main/screenEncodingCache.ts @@ -9,6 +9,8 @@ export interface ScreenEncodingRequest { readonly strategy: ScreenEncodingStrategy; /** Whether a software request also reports hardware availability, as Settings asks. */ readonly inspectHardware: boolean; + /** The compatible format (#858): only H.264 encoders, whatever the codec preference. */ + readonly h264Only?: boolean; } interface Entry { @@ -107,9 +109,9 @@ export class ScreenEncodingCache { } } -function cacheKey({ video, mode, codec, strategy, inspectHardware }: ScreenEncodingRequest): string { +function cacheKey({ video, mode, codec, strategy, inspectHardware, h264Only = false }: ScreenEncodingRequest): string { // Automatic discovery ignores dormant manual fields, and hardware discovery always inspects hardware. const automatic = strategy === 'automatic'; - return JSON.stringify([strategy, automatic ? 'hardware' : mode, automatic ? 'auto' : codec, + return JSON.stringify([strategy, automatic ? 'hardware' : mode, h264Only ? 'h264' : automatic ? 'auto' : codec, automatic || mode === 'hardware' || inspectHardware, video.width, video.height, video.fps, video.maxBitrateKbps]); } diff --git a/apps/client/src/main/screenEncodingPolicy.ts b/apps/client/src/main/screenEncodingPolicy.ts index 187757e7..e698487c 100644 --- a/apps/client/src/main/screenEncodingPolicy.ts +++ b/apps/client/src/main/screenEncodingPolicy.ts @@ -33,9 +33,12 @@ export async function selectScreenEncoding( (selection, _error, outcome) => { console.warn(`[ScreenShare] Hardware encoder probe failed (${outcome}):`, selection.encoder); }, + h264Only = false, ): Promise { - if (strategy === 'manual' && preference === 'auto') throw new Error('Manual encoding requires an exact codec.'); + if (strategy === 'manual' && preference === 'auto' && !h264Only) throw new Error('Manual encoding requires an exact codec.'); if (strategy === 'automatic') { mode = 'hardware'; preference = 'auto'; } + // The compatible format is H.264 Constrained Baseline (#858): AV1 is never a candidate. + if (h264Only) preference = 'h264'; let hardware: ScreenEncodingSelection | null = null; let hardwareError = false; let hardwareFailure: Error | null = null; diff --git a/apps/client/src/renderer/core/WebRtcManager.ts b/apps/client/src/renderer/core/WebRtcManager.ts index 7d2c995f..4abb4858 100644 --- a/apps/client/src/renderer/core/WebRtcManager.ts +++ b/apps/client/src/renderer/core/WebRtcManager.ts @@ -18,6 +18,7 @@ import { appEvents } from './EventBus'; import type { NetworkClient } from './NetworkClient'; import type { ParticipantManager } from './ParticipantManager'; import { callServerOrClosed, captureCall } from './callScope'; +import { createScreenShareStream } from './screenShareStream'; import { currentEventOrigin, emitOutsideRouting } from './sessionRouting'; import { clientLog } from './ClientLogService'; import { settingsStore } from '../stores/settingsStore'; @@ -258,7 +259,7 @@ export class WebRtcManager { throw new ScreenShareUserError(t('screenShare.nativeUnavailable')); if (!(capabilities.captureKinds ?? (capabilities.capture ? ['window'] : [])).includes(captureKind)) throw new ScreenShareUserError(t('screenShare.nativeUnavailable')); - const stream = new MediaStream(); + const stream = createScreenShareStream(); try { const source = await this.nativeScreens.addSource({ shareId: stream.id, desktopSourceId, captureKind, preserveAspectRatio, video, audio, audience, diff --git a/apps/client/src/renderer/core/screenEncoding.ts b/apps/client/src/renderer/core/screenEncoding.ts index d56b7432..c89c83da 100644 --- a/apps/client/src/renderer/core/screenEncoding.ts +++ b/apps/client/src/renderer/core/screenEncoding.ts @@ -31,7 +31,7 @@ export async function probeScreenEncoding( try { const result = await window.api.nativeScreenCommand({ action: 'probe-encoding', probeId, video, - ...preferences, + ...preferences, screenFormat: settingsStore.screenFormat, }); signal.throwIfAborted(); if (result.kind !== 'encoding') throw new Error('Invalid screen encoder discovery response.'); diff --git a/apps/client/src/renderer/core/screenShareStream.ts b/apps/client/src/renderer/core/screenShareStream.ts new file mode 100644 index 00000000..3d7b4559 --- /dev/null +++ b/apps/client/src/renderer/core/screenShareStream.ts @@ -0,0 +1,13 @@ +import { screenShareIdSchema } from '@monky/shared'; + +/** + * The stream a local screen share is shown through. Its id names the share everywhere (stores, IPC and the + * server), so it must be a valid share id. Firefox names every stream `{uuid}`, braces the rule refuses (#858): + * such a stream gets an id of its own, which only the app reads. + */ +export function createScreenShareStream(): MediaStream { + const stream = new MediaStream(); + if (!screenShareIdSchema.safeParse(stream.id).success) + Object.defineProperty(stream, 'id', { value: crypto.randomUUID(), enumerable: true }); + return stream; +} diff --git a/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts b/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts index 4eaf6cf2..d59e8ba2 100644 --- a/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts +++ b/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts @@ -170,9 +170,10 @@ export class BrowserScreenSfu { if (maxReceiveLevel) codec.parameters = { ...codec.parameters, 'max-recv-level': maxReceiveLevel }; } // A screen nobody here can decode is refused when consuming: by the server (no common profile) or by - // the browser's own description. Without an H.264 Main receiver that is the codec, not the network. + // the browser's own description. Without an H.264 Main receiver that is the codec, not the network; + // only a retirement (AbortError) stays what it is. const codecRefusal = (error: unknown): unknown => producer.kind === 'video' && !browserReceivesH264Main() - && !(error instanceof DOMException) + && !(error instanceof Error && error.name === 'AbortError') ? new BrowserScreenCodecError(`This browser cannot consume the ${selectedMime} screen: ${String(error)}`) : error; const consumed = await rpc(MessageType.SFU_CONSUME, { channelId: call.channelId, transportId: transport.id, producerId: producer.producerId, diff --git a/apps/client/src/renderer/core/webrtc/BrowserScreenSubscription.ts b/apps/client/src/renderer/core/webrtc/BrowserScreenSubscription.ts index 0e79fc0d..ea4447fe 100644 --- a/apps/client/src/renderer/core/webrtc/BrowserScreenSubscription.ts +++ b/apps/client/src/renderer/core/webrtc/BrowserScreenSubscription.ts @@ -5,7 +5,7 @@ import { } from '@monky/shared'; import { BrowserScreenP2p } from './BrowserScreenP2p'; import { BrowserScreenSfu, type BrowserScreenRpc } from './BrowserScreenSfu'; -import { BrowserScreenCodecError, supportsBrowserScreenCodec } from './browserScreenCodecs'; +import { BrowserScreenCodecError, browserScreenFailureReason, browserScreenRefusal, supportsBrowserScreenCodec } from './browserScreenCodecs'; export interface BrowserScreenSubscriptionOptions { call: NativeScreenCall; @@ -52,23 +52,30 @@ export class BrowserScreenSubscription { private fail(error: unknown): void { if (this.stopping) return; - this.options.onUnavailable(error instanceof BrowserScreenCodecError ? 'unsupported' : 'connection-failed'); + this.options.onUnavailable(browserScreenFailureReason(error)); this.options.onError(error); } /** * The first frame never came. Video that arrived yet never decoded is the browser's decoder refusing the * stream (Firefox accepts AV1 in its capability probe and then decodes none of a screen), not the network. + * So is a connection that carries media but no video at all, on a browser with no decoder for the codec + * the screen is in: it dropped the stream before counting it. */ private async startupExpired(message: string): Promise { - let undecoded = false; + let undecoded = false, videoPackets = 0, connected = false; try { (await this.stats())?.forEach(report => { - if (report.type === 'inbound-rtp' && report.kind === 'video' && report.packetsReceived > 0 && !report.framesDecoded) - undecoded = true; + if (report.type === 'inbound-rtp' && report.kind === 'video') { + videoPackets += report.packetsReceived ?? 0; + if (report.packetsReceived > 0 && !report.framesDecoded) undecoded = true; + } + if (report.type === 'candidate-pair' && report.state === 'succeeded' && report.bytesReceived > 0) connected = true; }); } catch { /* The stats only refine the reason. */ } - this.fail(undecoded ? new BrowserScreenCodecError(`${message} Its video arrived, but this browser decoded none of it.`) + const dropped = connected && videoPackets === 0 && browserScreenRefusal(this.options.source.codec ?? 'h264') === 'unwatchable'; + this.fail(undecoded || dropped ? new BrowserScreenCodecError(`${message} ${undecoded + ? 'Its video arrived, but this browser decoded none of it.' : 'The connection works, but this browser kept none of its video.'}`) : new Error(message)); } diff --git a/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts b/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts index 2170f130..b21c5d99 100644 --- a/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts +++ b/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts @@ -2,7 +2,17 @@ import { getScreenH264ProfileLevelId, getScreenAv1MinimumLevelIndex, type NativeScreenVideoProfile, type ScreenCodec, } from '@monky/shared'; -export class BrowserScreenCodecError extends Error {} +export class BrowserScreenCodecError extends Error { + override name = 'BrowserScreenCodecError'; +} + +/** + * Why a browser receiver failed (#858): a screen it cannot decode is `unsupported`, so the viewer is told the + * format and where to watch it; anything else is the connection. Read by name, as the error crosses modules. + */ +export function browserScreenFailureReason(error: unknown): 'unsupported' | 'connection-failed' { + return error instanceof Error && error.name === 'BrowserScreenCodecError' ? 'unsupported' : 'connection-failed'; +} const fmtpParameters = (value: string): Record => Object.fromEntries(value.split(';').flatMap(part => { const at = part.indexOf('='); @@ -65,15 +75,14 @@ export function browserReceivesScreenCodec(selected: ScreenCodec = 'h264'): bool } /** - * What to tell a browser about a screen in `codec` it could not decode: `main`, an H.264 Main share it lacks - * (Firefox, Safari); `switch`, another codec the publisher can change to the H.264 it receives; `codec`, a format - * it does not decode with nothing to switch to (Firefox and AV1); `null`, nothing codec-specific. + * What to tell a browser about a screen in `codec` it could not decode: `unwatchable`, a format it does not + * decode (Firefox and Safari facing H.264 Main or AV1), so it watches elsewhere or asks for the compatible + * format; `switch`, another codec the publisher can change to the H.264 it receives; `null`, nothing codec-specific. */ -export function browserScreenRefusal(codec: ScreenCodec): 'main' | 'switch' | 'codec' | null { +export function browserScreenRefusal(codec: ScreenCodec): 'unwatchable' | 'switch' | null { const main = browserReceivesH264Main(); - if (codec === 'h264' && !main && browserReceivesScreenCodec('h264')) return 'main'; if (codec !== 'h264' && main) return 'switch'; - if (!browserReceivesScreenCodec(codec) || !main) return 'codec'; + if (!main || !browserReceivesScreenCodec(codec)) return 'unwatchable'; return null; } diff --git a/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts b/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts index c1396fbf..7e739d72 100644 --- a/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts +++ b/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts @@ -8,7 +8,6 @@ import { emitOutsideRouting } from '../../sessionRouting'; import { t } from '../../../i18n'; import { videoService, type NativeScreenCapture } from '../../VideoService'; import { settingsStore } from '../../../stores/settingsStore'; -import { platformSupports } from '../../../platform/capabilities'; import { customVideoFpsLimit } from '../../../utils/qualityProfileLimits'; import { acceptScreenEncoding } from '../../screenEncoding'; import { @@ -18,8 +17,6 @@ import { /** Why viewers see a frozen picture of a local source: its window is minimized or hidden, or a browser stopped drawing it. */ export type NativeScreenSourceWarning = 'hidden' | 'covered'; -/** The format a browser publisher encodes in (#858); the desktop chooses its codec through its own settings. */ -const browserFormat = () => platformSupports('browserScreenPicker') ? { browserFormat: settingsStore.browserScreenFormat } : {}; export type SourceInput = Omit & { shareId: string; video: NativeScreenVideoProfile; audio: boolean; @@ -155,7 +152,7 @@ export class LocalScreenSources { desktopSourceId: input.desktopSourceId, video: input.video, audio: input.audio, audioBitrateKbps: input.audioBitrateKbps, preserveAspectRatio: input.preserveAspectRatio ?? true, encodingMode: settingsStore.screenEncodingMode, codec: settingsStore.preferredScreenCodec, - encodingStrategy: settingsStore.screenEncodingStrategy, ...browserFormat(), + encodingStrategy: settingsStore.screenEncodingStrategy, screenFormat: settingsStore.screenFormat, ...(input.replacesAudioShareIds?.length ? { replacesAudioShareIds: [...input.replacesAudioShareIds] } : {}), ...(input.captureKind ? { captureKind: input.captureKind } : {}), }).then(result => { @@ -225,7 +222,7 @@ export class LocalScreenSources { preserveAspectRatio: capture.preserveAspectRatio ?? true, video, audio, audioBitrateKbps, encodingMode: settingsStore.screenEncodingMode, codec: settingsStore.preferredScreenCodec, - encodingStrategy: settingsStore.screenEncodingStrategy, ...browserFormat(), + encodingStrategy: settingsStore.screenEncodingStrategy, screenFormat: settingsStore.screenFormat, }); this.calls.current(call); if (!isCurrentCapture() || previous.removing || call.sources.get(shareId) !== previous) { diff --git a/apps/client/src/renderer/core/webrtc/nativeScreen/RemoteScreenPresentations.ts b/apps/client/src/renderer/core/webrtc/nativeScreen/RemoteScreenPresentations.ts index 13b74ed1..a7c16d32 100644 --- a/apps/client/src/renderer/core/webrtc/nativeScreen/RemoteScreenPresentations.ts +++ b/apps/client/src/renderer/core/webrtc/nativeScreen/RemoteScreenPresentations.ts @@ -9,6 +9,7 @@ import { voiceStore } from '../../../stores/voiceStore'; import { resolveAudioOutput } from '../../../utils/audioPreferences'; import { showInfoToast } from '../../../views/CopyToast'; import { BrowserScreenSubscription } from '../BrowserScreenSubscription'; +import { browserScreenFailureReason } from '../browserScreenCodecs'; import type { RemoteMediaRouter } from '../RemoteMediaRouter'; import { announceScreenChange, cancelled, keyOf, messageOf, reportScreenError, @@ -253,7 +254,8 @@ export class RemoteScreenPresentations { public browserFailed(call: Call, entry: Presentation, error: unknown): void { if (entry.stopping) return; reportScreenError(error); - entry.state = { state: 'unavailable', reason: 'connection-failed' }; + // A codec the browser cannot decode is not the network: the viewer is told the format (#858). + entry.state = { state: 'unavailable', reason: browserScreenFailureReason(error) }; this.rewatch(call).outcome(keyOf(entry.publisherSessionId, entry.source.shareId), entry); announceScreenChange(); void this.stop(call, entry).catch(reportScreenError); diff --git a/apps/client/src/renderer/i18n/locales/en.ts b/apps/client/src/renderer/i18n/locales/en.ts index afa69f28..6b920a43 100644 --- a/apps/client/src/renderer/i18n/locales/en.ts +++ b/apps/client/src/renderer/i18n/locales/en.ts @@ -957,14 +957,14 @@ export const en: TranslationMap = { 'screenShare.nativeReceiverUnavailable': 'The native receiver is unavailable. Monky will not switch to Chromium automatically.', 'screenShare.nativeReceiverSettingsHint': 'You can select Chromium in Settings → Quality & sharing → Screen reception. Read the limitations before switching, then try watching again.', 'screenShare.chromiumReceiverSettingsHint': 'You are using the Chromium receiver, which cannot receive AV1 at high-resolution profiles. Select Native in Settings → Quality & sharing → Screen reception and try watching again, or ask the publisher to use H.264.', - 'screenShare.browserReceiverHint': 'Ask the publisher to use H.264 and try watching again.', - 'screenShare.browserCannotReceive': 'This browser cannot receive {codec} screens, the format of this share. Watch it in Chrome, Edge or the desktop app.', - 'screenShare.browserCannotReceiveMain': 'This browser cannot receive this share: it uses H.264 Main, from the desktop app or the web\'s better quality option. Watch it in Chrome, Edge or the desktop app.', - 'screenShare.browserFormat': 'Video format', - 'screenShare.browserFormatCompatible': 'Compatible', - 'screenShare.browserFormatCompatibleDescription': 'H.264 everyone can watch: Chrome, Edge, Firefox, Safari and the desktop app. Recommended.', - 'screenShare.browserFormatMain': 'Better quality', - 'screenShare.browserFormatMainDescription': 'H.264 Main: a better picture with the same bandwidth. Firefox and Safari cannot watch it.', + 'screenShare.browserReceiverHint': 'Ask the publisher to use the Compatible format and try watching again.', + 'screenShare.browserCannotWatch': 'This browser cannot watch this share. Ask the publisher to use the Compatible format, or watch it in Chrome, Edge or the desktop app.', + 'screenShare.format': 'Video format', + 'screenShare.formatCompatible': 'Compatible', + 'screenShare.formatCompatibleDescription': 'Everyone can watch it: Chrome, Edge, Firefox, Safari and the desktop app. Recommended.', + 'screenShare.formatMain': 'Better quality', + 'screenShare.formatMainDescription': 'A better picture with the same connection, but people on Firefox or Safari cannot watch it.', + 'settings.screenFormatApply': 'Applies from the next screen share. In the Compatible format, the screen is always sent in H.264.', 'desktopOnly.badge': 'Desktop', 'desktopOnly.tooltip': 'Available in the desktop app', 'desktopOnly.reason.screenShare': 'This browser cannot share screens to Monky. Use Chrome, Edge or Firefox, or the desktop app.', diff --git a/apps/client/src/renderer/i18n/locales/pt-BR.ts b/apps/client/src/renderer/i18n/locales/pt-BR.ts index 95a5b8d1..23d848ef 100644 --- a/apps/client/src/renderer/i18n/locales/pt-BR.ts +++ b/apps/client/src/renderer/i18n/locales/pt-BR.ts @@ -919,14 +919,14 @@ export const ptBR = { 'screenShare.nativeReceiverUnavailable': 'O receptor nativo não está disponível. Nenhuma troca automática para Chromium será feita.', 'screenShare.nativeReceiverSettingsHint': 'Você pode selecionar Chromium em Configurações → Qualidade e compartilhamento → Recepção de tela. Leia o aviso de limitações antes de trocar e tente assistir novamente.', 'screenShare.chromiumReceiverSettingsHint': 'Você está usando o receptor Chromium, que não recebe AV1 em perfis de alta resolução. Selecione Nativo em Configurações → Qualidade e compartilhamento → Recepção de tela e tente assistir novamente, ou peça ao transmissor para usar H.264.', - 'screenShare.browserReceiverHint': 'Peça a quem está transmitindo para usar H.264 e tente assistir novamente.', - 'screenShare.browserCannotReceive': 'Este navegador não recebe telas em {codec}, o formato desta transmissão. Assista pelo Chrome, pelo Edge ou pelo app desktop.', - 'screenShare.browserCannotReceiveMain': 'Este navegador não recebe esta transmissão: ela usa H.264 Main, do app desktop ou da opção de melhor qualidade da web. Assista pelo Chrome, pelo Edge ou pelo app desktop.', - 'screenShare.browserFormat': 'Formato do vídeo', - 'screenShare.browserFormatCompatible': 'Compatível', - 'screenShare.browserFormatCompatibleDescription': 'H.264 que todos assistem: Chrome, Edge, Firefox, Safari e o app desktop. Recomendado.', - 'screenShare.browserFormatMain': 'Melhor qualidade', - 'screenShare.browserFormatMainDescription': 'H.264 Main: imagem melhor com a mesma banda. O Firefox e o Safari não conseguem assistir.', + 'screenShare.browserReceiverHint': 'Peça a quem está transmitindo para usar o formato Compatível e tente assistir novamente.', + 'screenShare.browserCannotWatch': 'Este navegador não consegue assistir a esta transmissão. Peça a quem está transmitindo para usar o formato Compatível, ou assista pelo Chrome, pelo Edge ou pelo app desktop.', + 'screenShare.format': 'Formato do vídeo', + 'screenShare.formatCompatible': 'Compatível', + 'screenShare.formatCompatibleDescription': 'Todo mundo consegue assistir: Chrome, Edge, Firefox, Safari e o app desktop. Recomendado.', + 'screenShare.formatMain': 'Melhor qualidade', + 'screenShare.formatMainDescription': 'Imagem melhor com a mesma internet, mas quem usa o Firefox ou o Safari não consegue assistir.', + 'settings.screenFormatApply': 'Vale a partir do próximo compartilhamento de tela. No formato Compatível, a tela sai sempre em H.264.', 'desktopOnly.badge': 'Desktop', 'desktopOnly.tooltip': 'Disponível no app desktop', 'desktopOnly.reason.screenShare': 'Este navegador não consegue transmitir tela para o Monky. Use o Chrome, o Edge ou o Firefox, ou o app desktop.', diff --git a/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts b/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts index 48283b49..4fae9236 100644 --- a/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts +++ b/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts @@ -1,4 +1,4 @@ -import type { BrowserScreenFormat, NativeScreenCall } from '@monky/shared'; +import type { ScreenFormat, NativeScreenCall } from '@monky/shared'; import { screenSenderCodecs } from './browserScreenProfile'; import { sentMediaBinding, withScreenStreamId, type ScreenMediaBinding } from './screenSdp'; import type { ScreenIceCandidate, ScreenMediaKind, ScreenPeerPort } from './screenP2pPublisherPeer'; @@ -6,7 +6,7 @@ import type { ScreenIceCandidate, ScreenMediaKind, ScreenPeerPort } from './scre export interface BrowserScreenPeerOptions { readonly iceServers: NativeScreenCall['iceServers']; /** The only video codecs this connection may send (#858). */ - readonly format: BrowserScreenFormat; + readonly format: ScreenFormat; /** The screen's instance: the stream id viewers group its video and sound by. */ readonly streamId: string; readonly video: MediaStreamTrack; diff --git a/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts b/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts index 7093535a..88d24671 100644 --- a/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts +++ b/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts @@ -1,6 +1,6 @@ import { - browserScreenFormatSchema, getScreenShareProfile, nativeScreenVideoProfileSchema, - type BrowserScreenFormat, type NativeScreenVideoProfile, + screenFormatSchema, getScreenShareProfile, nativeScreenVideoProfileSchema, + type ScreenFormat, type NativeScreenVideoProfile, } from '@monky/shared'; /** What a captured video track reports of itself (`MediaStreamTrack.getSettings()`). */ @@ -45,7 +45,7 @@ const fmtp = (codec: CodecCapability, key: string): string | undefined => * Whether a sender codec encodes a browser format (#858): H.264 in non-interleaved mode, as Constrained * Baseline (42e0..) for `compatible`, which every receiver decodes, or as Main (4d..) for `main`. */ -export function isScreenCodec(codec: CodecCapability, format: BrowserScreenFormat): boolean { +export function isScreenCodec(codec: CodecCapability, format: ScreenFormat): boolean { const profile = format === 'main' ? /^4d[0-9a-f]{4}$/i : /^42e0[0-9a-f]{2}$/i; return codec.mimeType.toLowerCase() === 'video/h264' && profile.test(fmtp(codec, 'profile-level-id') ?? '') && fmtp(codec, 'packetization-mode') === '1'; @@ -55,13 +55,13 @@ export function isScreenCodec(codec: CodecCapability, format: BrowserScreenForma * The video codecs a browser screen sender offers for `format`, in order: the format's codec and its * retransmission. An empty list means this browser cannot share in that format. */ -export function screenSenderCodecs(codecs: readonly T[], format: BrowserScreenFormat): T[] { +export function screenSenderCodecs(codecs: readonly T[], format: ScreenFormat): T[] { const matching = codecs.filter(codec => isScreenCodec(codec, format)); if (!matching.length) return []; return [...matching, ...codecs.filter(codec => codec.mimeType.toLowerCase() === 'video/rtx')]; } /** The formats this browser can share in, the compatible one first. */ -export function browserScreenFormats(codecs: readonly CodecCapability[]): BrowserScreenFormat[] { - return browserScreenFormatSchema.options.filter(format => screenSenderCodecs(codecs, format).length > 0); +export function browserScreenFormats(codecs: readonly CodecCapability[]): ScreenFormat[] { + return screenFormatSchema.options.filter(format => screenSenderCodecs(codecs, format).length > 0); } diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts index 13a7db00..6fcf4988 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts @@ -1,7 +1,7 @@ import { getScreenShareProfile, nativeScreenFailureReason, nativeScreenSignalSchema, screenShareProfileKey, type NativeScreenCall, type NativeScreenFailure, type NativeScreenSignalPayload, type NativeScreenSource, - type NativeScreenVideoProfile, type ScreenShareQuality, type BrowserScreenFormat, + type NativeScreenVideoProfile, type ScreenShareQuality, type ScreenFormat, } from '@monky/shared'; import { BrowserScreenPeer } from './browserScreenPeer'; import type { DisplayCapture } from './displayCaptures'; @@ -49,7 +49,7 @@ export interface WebScreenPublisherOptions { readonly call: NativeScreenCall; readonly source: NativeScreenSource; /** How this screen is encoded; its codec is the source's. */ - readonly format: BrowserScreenFormat; + readonly format: ScreenFormat; readonly capture: DisplayCapture; readonly audioBitrateKbps: number; readonly signal: (signal: NativeScreenSignalPayload) => Promise; diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts index 32b61d3b..e666c036 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts @@ -1,5 +1,5 @@ import { - SCREEN_SHARE_LIMITS, browserScreenCommandSchema, nativeScreenCallSchema, nativeScreenPresentationSchema, + SCREEN_SHARE_LIMITS, browserScreenCommandSchema, compatibleScreenProfile, nativeScreenCallSchema, nativeScreenPresentationSchema, nativeScreenSignalSchema, nativeScreenSourceSchema, type NativeScreenCall, type NativeScreenCapabilities, type NativeScreenCommandResult, type NativeScreenEvent, type NativeScreenParticipant, type NativeScreenPresentation, type NativeScreenSignalPayload, type NativeScreenSource, @@ -242,7 +242,7 @@ export class WebScreenRuntime { if (command.audio && (!capture.audioTrack || !capabilities.captureAudio)) throw new Error('The browser did not share sound with this capture.'); // The compatible format unless the publisher chose a better one this browser encodes; both are H.264. - const format = command.browserFormat ?? 'compatible'; + const format = command.screenFormat ?? 'compatible'; if (!capabilities.browserFormats?.includes(format)) throw new Error(`This browser cannot share in the ${format} format.`); call.admitting.add(command.shareId); try { @@ -251,7 +251,8 @@ export class WebScreenRuntime { if (capture.ended) throw sourceLost(); const source = nativeScreenSourceSchema.parse({ shareId: command.shareId, instanceId: crypto.randomUUID(), - video: browserScreenProfile(command.video, capture.video.getSettings()), audio: command.audio, codec: 'h264', + video: format === 'compatible' ? compatibleScreenProfile(browserScreenProfile(command.video, capture.video.getSettings())) + : browserScreenProfile(command.video, capture.video.getSettings()), audio: command.audio, codec: 'h264', }); const publisher = new WebScreenPublisher({ call: call.config, source, format, capture, audioBitrateKbps: command.audioBitrateKbps, diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts index 8da00b98..64824d45 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts @@ -1,5 +1,5 @@ import { Device, type types as SfuTypes } from 'mediasoup-client'; -import { MessageType, type BrowserScreenFormat, type NativeScreenRendition } from '@monky/shared'; +import { MessageType, type ScreenFormat, type NativeScreenRendition } from '@monky/shared'; import { isScreenCodec } from './browserScreenProfile'; export type WebScreenRpc = (method: MessageType, payload: Record) => Promise; @@ -7,7 +7,7 @@ export type WebScreenRpc = (method: MessageType, payload: Record platformSupports('browserScreenPicker') ? 'compatible' : 'main'; + /** * Chat-notification-sound mode for the 3-level configuration (#153). * - `inherit`: fall back to the next level up (channel → server → global). @@ -53,7 +55,7 @@ function savedSettings(store: SettingsStore) { screenEncodingMode: store.screenEncodingMode, screenEncodingStrategy: store.screenEncodingStrategy, preferredScreenCodec: store.preferredScreenCodec, - browserScreenFormat: store.browserScreenFormat, + screenFormat: store.screenFormat, vadSensitivity: store.vadSensitivity, inputVolume: store.inputVolume, outputVolume: store.outputVolume, @@ -205,8 +207,8 @@ export class SettingsStore { public screenEncodingMode: ScreenEncodingMode = 'hardware'; public screenEncodingStrategy: ScreenEncodingStrategy = 'automatic'; public preferredScreenCodec: ScreenCodec = 'h264'; - /** How the web version encodes the screens it shares (#858): compatible with every viewer by default. */ - public browserScreenFormat: BrowserScreenFormat = 'compatible'; + /** The H.264 screens are shared in (#858): every viewer watches the web's by default; the desktop keeps Main. */ + public screenFormat: ScreenFormat = defaultScreenFormat(); public customProfile: QualityProfile = { ...DEFAULT_CUSTOM_PROFILE }; public inputMode: 'voice_activity' | 'push_to_talk' = 'voice_activity'; // #186 public pttKey: PttKeyBinding = { code: 'KeyV', display: 'V', keyType: 'keyboard', keyCode: 47 }; @@ -330,7 +332,7 @@ export class SettingsStore { this.screenEncodingMode = 'hardware'; this.screenEncodingStrategy = 'automatic'; this.preferredScreenCodec = 'h264'; - this.browserScreenFormat = 'compatible'; + this.screenFormat = defaultScreenFormat(); this.screenShareReceiver = 'native'; this.soundboardLimiterEnabled = false; this.soundboardLoudnessLimit = 6; @@ -355,8 +357,9 @@ export class SettingsStore { this.qualityPreset = restoreQualityPreset(parsed.qualityPreset); this.screenEncodingMode = parsed.screenEncodingMode === 'software' ? 'software' : 'hardware'; this.preferredScreenCodec = parsed.preferredScreenCodec === 'av1' ? 'av1' : 'h264'; - const browserFormat = browserScreenFormatSchema.safeParse(parsed.browserScreenFormat); - this.browserScreenFormat = browserFormat.success ? browserFormat.data : 'compatible'; + // v41.5.4 saved the web's choice as browserScreenFormat. + const screenFormat = screenFormatSchema.safeParse(parsed.screenFormat ?? parsed.browserScreenFormat); + this.screenFormat = screenFormat.success ? screenFormat.data : defaultScreenFormat(); this.screenEncodingStrategy = parsed.screenEncodingStrategy === 'manual' ? 'manual' : parsed.screenEncodingStrategy === undefined && (parsed.screenEncodingMode === 'software' || parsed.preferredScreenCodec === 'h264' @@ -921,8 +924,8 @@ export class SettingsStore { catch (error) { this.screenShareReceiver = previous; throw error; } } - public setBrowserScreenFormat(format: BrowserScreenFormat): void { - this.browserScreenFormat = browserScreenFormatSchema.parse(format); + public setScreenFormat(format: ScreenFormat): void { + this.screenFormat = screenFormatSchema.parse(format); this.save(); } diff --git a/apps/client/src/renderer/views/ScreenEncodingControls.ts b/apps/client/src/renderer/views/ScreenEncodingControls.ts index 1c364fb0..ccb29bf8 100644 --- a/apps/client/src/renderer/views/ScreenEncodingControls.ts +++ b/apps/client/src/renderer/views/ScreenEncodingControls.ts @@ -183,15 +183,19 @@ export class ScreenEncodingControls { }, { signal: this.events?.signal }); } + // The compatible format is always H.264 (#858): the codec choice waits for the better-quality format. private savedPreferences(): ScreenEncodingPreferences { - return { encodingStrategy: settingsStore.screenEncodingStrategy, - encodingMode: settingsStore.screenEncodingMode, codec: settingsStore.preferredScreenCodec }; + return { encodingStrategy: settingsStore.screenEncodingStrategy, encodingMode: settingsStore.screenEncodingMode, + codec: settingsStore.screenFormat === 'compatible' ? 'h264' : settingsStore.preferredScreenCodec }; } - private preferences(): ScreenEncodingPreferences { return this.requested ?? this.savedPreferences(); } + private preferences(): ScreenEncodingPreferences { + const preferences = this.requested ?? this.savedPreferences(); + return settingsStore.screenFormat === 'compatible' ? { ...preferences, codec: 'h264' } : preferences; + } private key(profile = this.profile(), preferences = this.preferences()): string { - return JSON.stringify([profile, preferences.encodingStrategy, + return JSON.stringify([profile, settingsStore.screenFormat, preferences.encodingStrategy, ...(preferences.encodingStrategy === 'automatic' ? [] : [preferences.encodingMode, preferences.codec])]); } @@ -440,7 +444,8 @@ export class ScreenEncodingControls { this.render(); try { for (const mode of ['hardware', 'software'] as const) { - for (const codec of ['h264', 'av1'] as const) { + // The compatible format has no AV1 to check (#858). + for (const codec of settingsStore.screenFormat === 'compatible' ? ['h264'] as const : ['h264', 'av1'] as const) { if (!current() || defer()) return; if (this.choice(mode, codec)) continue; let result: CheckedEncoding; @@ -499,13 +504,15 @@ export class ScreenEncodingControls { } const codec = this.container?.querySelector('#select-video-codec'); if (codec) { - codec.disabled = unavailable || automatic; - codec.setAttribute('aria-readonly', String(unavailable || automatic)); + const compatible = settingsStore.screenFormat === 'compatible'; + codec.disabled = unavailable || automatic || compatible; + codec.setAttribute('aria-readonly', String(unavailable || automatic || compatible)); for (const value of ['h264', 'av1'] as const) { const option = codec.querySelector(`option[value="${value}"]`); if (!option) continue; const choice = this.choice(preferences.encodingMode, value); - option.disabled = !automatic && !!choice && !('error' in choice) && !choice.availability.selection; + option.disabled = (compatible && value !== 'h264') + || (!automatic && !!choice && !('error' in choice) && !choice.availability.selection); const label = t(value === 'av1' ? 'settings.codecAv1' : 'settings.codecH264'); const explanation = automatic ? '' : this.choiceText(preferences.encodingMode, value, choice); option.textContent = !automatic && (!choice || 'error' in choice || !choice.availability.selection) @@ -514,7 +521,7 @@ export class ScreenEncodingControls { ? t('settings.screenEncodingRetry') : t('screenShare.unavailable')}` : label; option.title = explanation; } - codec.value = automatic ? selection?.codec ?? '' : preferences.codec; + codec.value = compatible ? 'h264' : automatic ? selection?.codec ?? '' : preferences.codec; if (!automatic && codec.querySelector(`option[value="${preferences.codec}"]`)?.disabled) codec.value = ''; } for (const mode of ['hardware', 'software'] as const) { diff --git a/apps/client/src/renderer/views/VoiceStageView.ts b/apps/client/src/renderer/views/VoiceStageView.ts index fa630164..908c41a8 100644 --- a/apps/client/src/renderer/views/VoiceStageView.ts +++ b/apps/client/src/renderer/views/VoiceStageView.ts @@ -1225,11 +1225,9 @@ export class VoiceStageView { if (state.reason !== 'unsupported') return failure; // Choosing the native receiver exists only where the platform has one (the desktop). if (platformSupports('nativeScreenAudio')) return `${failure} ${t('screenShare.chromiumReceiverSettingsHint')}`; - const codec = webRtcManager.getNativeScreenSource(sessionId, shareId)?.codec ?? 'h264'; - switch (browserScreenRefusal(codec)) { - case 'main': return t('screenShare.browserCannotReceiveMain'); + switch (browserScreenRefusal(webRtcManager.getNativeScreenSource(sessionId, shareId)?.codec ?? 'h264')) { + case 'unwatchable': return t('screenShare.browserCannotWatch'); case 'switch': return `${failure} ${t('screenShare.browserReceiverHint')}`; - case 'codec': return t('screenShare.browserCannotReceive', { codec: codec === 'av1' ? 'AV1' : 'H.264' }); default: return failure; } } diff --git a/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts b/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts index a57f3c21..83b568d6 100644 --- a/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts +++ b/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts @@ -1,4 +1,4 @@ -import { PUBLIC_AUDIENCE, browserScreenFormatSchema, type BrowserScreenFormat, type ResourceAudience } from '@monky/shared'; +import { PUBLIC_AUDIENCE, screenFormatSchema, type ScreenFormat, type ResourceAudience } from '@monky/shared'; import { html, trustedHtml } from '../../utils/html'; import { appEvents } from '../../core/EventBus'; import { captureScreenShareCall, screenShareLimit, type ScreenShareCall } from '../../core/screenShareControls'; @@ -9,7 +9,8 @@ import { nativeScreenProfile } from '../../core/webrtc/NativeScreenController'; import { voiceStore } from '../../stores/voiceStore'; import { setButtonLoading } from '../../utils/buttonLoading'; import { openModal, type ModalShell } from '../../utils/modalShell'; -import { renderChoiceCards, renderSwitch, selectChoiceCard, type ChoiceCard } from '../../utils/formControls'; +import { renderChoiceCards, renderSwitch, selectChoiceCard } from '../../utils/formControls'; +import { screenFormatChoice } from './screenFormatChoices'; import { settingsStore } from '../../stores/settingsStore'; import { ScreenShareUserError, screenShareFailureMessage } from '../../utils/screenShareFailure'; import { t } from '../../i18n'; @@ -28,13 +29,6 @@ const SCREEN_AUDIENCE_TEXTS: AudienceTexts = { members: 'screenShare.privateUsers', more: 'screenShare.privateMore', }; -/** Each format a browser shares in, said in its own words: who can watch it is part of the choice. */ -const FORMAT_CHOICES: Readonly ChoiceCard>> = { - compatible: () => ({ value: 'compatible', icon: 'devices', title: t('screenShare.browserFormatCompatible'), - description: t('screenShare.browserFormatCompatibleDescription') }), - main: () => ({ value: 'main', icon: 'high_quality', title: t('screenShare.browserFormatMain'), - description: t('screenShare.browserFormatMainDescription') }), -}; function liveScreens(): LiveScreens { return { @@ -58,7 +52,7 @@ export class BrowserScreenSharePicker { private capabilitiesKnown = false; private available = false; /** The formats this browser can share in, the compatible one first. */ - private formats: readonly BrowserScreenFormat[] = []; + private formats: readonly ScreenFormat[] = []; private starting = false; private createAudience(initial: ResourceAudience): ResourceAudiencePicker { @@ -97,7 +91,7 @@ export class BrowserScreenSharePicker { width: profile.width, height: profile.height, fps: profile.fps, bitrate: profile.maxBitrateKbps, }) : t('screenShare.nativeProfileChangeBlocked')}