Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,17 @@ void VerifyAdaptiveDecoderSessions() {
Check(amf_plan && amf_plan->profileLevelId == "4d0433" && amf_info.sps->compatibility == 0x04,
"AMF constrained Main SPS must be admitted without rewriting its original bytes");
Reject([&] { plan(std::nullopt, amf_info, amf_packet); }, "ERR_RTC_NEGOTIATED_SPS");
// A browser's software encoder answers Main SDP with Main-compatible Constrained Baseline (42c0).
const auto browser_plan = policy::PlanDecoderSession(std::nullopt, small_info, smaller,
{screen_video::H264Profile::Main, 51}, 1920, 1080, 8, options);
Check(browser_plan && browser_plan->profileLevelId == "42c033",
"A Main receiver must admit the Main-compatible Constrained Baseline of browser software encoders");
auto plain_baseline = small_sps;
plain_baseline[2] = 0x80;
const auto plain_packet = ParserPacket(plain_baseline);
Reject([&] { policy::PlanDecoderSession(std::nullopt, policy::InspectAccessUnit(plain_packet), plain_packet,
{screen_video::H264Profile::Main, 51}, 1920, 1080, 8, options); },
"ERR_RTC_NEGOTIATED_SPS");
for (const auto flags : {0x08, 0x0c, 0x20, 0x80}) {
auto unsupported = *amf_info.sps;
unsupported.compatibility = static_cast<std::uint8_t>(flags);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ int32_t DecodeLocked(const webrtc::EncodedImage& image, int64_t render_time_ms)
if (sps) {
Require(sps->width >= 4 && sps->width <= 3840 && sps->height >= 2 && sps->height <= 2160 &&
!(sps->width & 1) && !(sps->height & 1) && sps->levelIdc <= mode_.level &&
sv::MatchesH264Profile(*sps, mode_.profile) && sv::IsBt709LimitedCompatible(*sps),
sv::DecodesAsH264Profile(*sps, mode_.profile) && sv::IsBt709LimitedCompatible(*sps),
"ERR_RTC_MAC_DECODER_SPS");
if (width_ != sps->width || height_ != sps->height) {
Require(keyframe, "ERR_RTC_MAC_DECODER_RESIZE");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,22 @@ AccessUnitInfo InspectAccessUnit(std::span<const std::uint8_t> bytes) {

void ValidateSps(const sv::H264Sps& sps, NegotiatedH264 negotiated,
std::uint32_t maximum_width, std::uint32_t maximum_height) {
// Under Main SDP, browsers' software encoder (OpenH264) sends Main-compatible Constrained Baseline.
const bool main_compatible_baseline = negotiated.profile == sv::H264Profile::Main &&
sps.profileIdc == 66 && sv::DecodesAsH264Profile(sps, negotiated.profile);
if (!sps.progressive || sps.width < 16 || sps.height < 16 ||
sps.width > 3840 || sps.height > 2160 ||
sps.width > maximum_width || sps.height > maximum_height ||
((sps.width | sps.height | sps.cropLeft | sps.cropTop) & 1) ||
!IsSupportedLevel(sps.levelIdc) || sps.levelIdc > negotiated.level ||
!sv::MatchesH264Profile(sps, negotiated.profile) ||
!sv::DecodesAsH264Profile(sps, negotiated.profile) ||
(sps.profileIdc == 66 && !(sps.compatibility & 0x40)) ||
!sv::IsBt709LimitedCompatible(sps)) {
throw AdapterError("ERR_RTC_NEGOTIATED_SPS",
"Actual SPS violates negotiated profile/level, geometry, or BT.709-limited input");
}
const auto actual = webrtc::ParseH264ProfileLevelId(sps.ProfileLevelId().c_str());
const auto expected = negotiated.profile == sv::H264Profile::Main
const auto expected = negotiated.profile == sv::H264Profile::Main && !main_compatible_baseline
? webrtc::H264Profile::kProfileMain : webrtc::H264Profile::kProfileConstrainedBaseline;
// Pinned AMF emits this Main SPS byte pattern, outside WebRTC's SDP table.
// Retain the original SPS; validate geometry and I/P pictures independently
Expand Down Expand Up @@ -178,7 +181,9 @@ std::optional<sv::DecoderConfig> PlanDecoderSession(
config.maxInFlight > 256ull * 1024 * 1024)
throw AdapterError("ERR_RTC_DECODER_MODE",
"Actual SPS does not fit configured MF rate/bitrate hints or GPU budget");
sv::H264Bitstream initial(sps.width, sps.height, sps.levelIdc, negotiated.profile);
// Qualify the IDR by what the SPS declares: Main, or the Main-compatible Constrained Baseline ValidateSps admitted.
const auto declared = sv::MatchesH264Profile(sps, negotiated.profile) ? negotiated.profile : sv::H264Profile::ConstrainedBaseline;
sv::H264Bitstream initial(sps.width, sps.height, sps.levelIdc, declared);
const auto checked = initial.Convert(packet);
if (!checked.hasPicture || !checked.keyFrame)
throw AdapterError("ERR_RTC_INITIAL_IDR", "The replacement session requires a complete qualified IDR");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ bool MatchesH264Profile(const H264Sps& sps, H264Profile profile) {
return false;
}

bool DecodesAsH264Profile(const H264Sps& sps, H264Profile negotiated) {
return MatchesH264Profile(sps, negotiated) ||
(negotiated == H264Profile::Main && MatchesH264Profile(sps, H264Profile::ConstrainedBaseline));
}

H264Sps ParseH264Sps(std::span<const std::uint8_t> nal) {
if (Type(nal) != 7 || nal.size() > maxParameterBytes) throw std::runtime_error("Invalid SPS");
Bits bits(nal);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ H264Sps ParseBaselineSps(std::span<const std::uint8_t> nal);
H264Sps ParseH264Sps(std::span<const std::uint8_t> nal);
H264Sps ParseProfileLevelId(const std::string& value);
bool MatchesH264Profile(const H264Sps& sps, H264Profile profile);
// What a receiver of the negotiated profile decodes: the profile itself and, for Main, the Constrained
// Baseline that declares Main conformance (constraint_set1, H.264 A.2.1), as browsers' OpenH264 sends.
bool DecodesAsH264Profile(const H264Sps& sps, H264Profile negotiated);
bool IsBt709LimitedCompatible(const H264Sps& sps);

class H264Bitstream {
Expand Down
2 changes: 1 addition & 1 deletion apps/client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"build:preload": "tsc -p tsconfig.preload.json && node scripts/buildPreloads.mjs",
"start": "electron .",
"screenshots:onboarding": "node scripts/capture-onboarding-screenshots.cjs",
"test": "tsc -p tsconfig.test.json && node dist-test/test/utils.test.js && node --test dist-test/test/eventBus.test.js dist-test/test/featureAvailability.test.js dist-test/test/botInteractions.test.js dist-test/test/botScreens.test.js dist-test/test/commandUi.test.js dist-test/test/commandAutocomplete.test.js dist-test/test/soundDownloads.test.js dist-test/test/soundDownloadConfirmation.test.js dist-test/test/presence.test.js dist-test/test/gameShare.test.js dist-test/test/rolePermissions.test.js dist-test/test/networkClient.test.js dist-test/test/serverShutdownNotice.test.js dist-test/test/voiceConnection.test.js dist-test/test/voiceModeReconnect.test.js dist-test/test/webRtcManager.test.js dist-test/test/screenSharing.test.js dist-test/test/screenShareControls.test.js dist-test/test/screenShareFailures.test.js dist-test/test/screenSubscriptions.test.js dist-test/test/webScreenPublisher.test.js dist-test/test/videoDiagnostics.test.js dist-test/test/videoTelemetry.test.js dist-test/test/performanceDiagnostics.test.js dist-test/test/responsivenessMonitor.test.js dist-test/test/shortcuts.test.js dist-test/test/releaseNotes.test.js dist-test/test/audioPreferences.test.js dist-test/test/serverSettings.test.js dist-test/test/voiceRooms.test.js dist-test/test/favorites.test.js dist-test/test/cameraEffects.test.js dist-test/test/screenColorPicker.test.js dist-test/test/serverSettingsStore.test.js dist-test/test/hostedServerLifecycle.test.js dist-test/test/messageEditing.test.js dist-test/test/messageSelection.test.js dist-test/test/markdownFormatting.test.js dist-test/test/serverEventCalendar.test.js dist-test/test/autoEntry.test.js dist-test/test/hostedServers.test.js dist-test/test/serverAddress.test.js dist-test/test/backgroundRouting.test.js dist-test/test/nicknames.test.js dist-test/test/livePulse.test.js dist-test/test/botLocalePreferencesStore.test.js dist-test/test/serverMonitor.test.js dist-test/test/serverAudit.test.js dist-test/test/sessionBinding.test.js dist-test/test/localPermissions.test.js dist-test/test/botDeviceStorage.test.js dist-test/test/localExecutionService.test.js dist-test/test/localTools.test.js dist-test/test/localExecutionRenderer.test.js dist-test/test/localExecutionTasks.test.js dist-test/test/localAudioSender.test.js dist-test/test/worker.test.js dist-test/test/workerClient.test.js dist-test/test/workerProtocol.test.js dist-test/test/mediaFolders.test.js dist-test/test/attachmentDownload.test.js dist-test/test/localPreparationView.test.js test/serverManagerLifecycle.test.cjs test/globalInputHookProcess.test.cjs test/developmentProfile.test.cjs test/updaterCompatibility.test.cjs test/crashRecovery.test.cjs test/serverEventCalendarIpc.test.cjs test/hostingSkills.test.cjs test/fileBatchDownloadIpc.test.cjs test/identityExport.test.cjs test/identityService.test.cjs test/localExecutionIpc.test.cjs test/localToolsTab.test.cjs test/steamPresence.test.cjs test/taskbarCallControls.test.cjs test/screenSharingUi.test.cjs test/soundboardLimiter.test.cjs test/trayManager.test.cjs test/onboardingShots.test.cjs test/ipcSenderPolicy.test.cjs test/windowSandbox.test.cjs test/recentSoundSaveIpc.test.cjs test/webDeployment.test.cjs test/zipFormat.test.cjs dist-test/test/settingsPersistence.test.js test/startupLoading.test.cjs dist-test/test/pointerReorder.test.js dist-test/test/rovingFocus.test.js && node --test dist-test/test/authHardening.test.js dist-test/test/hardeningAndOverlay.test.js dist-test/test/dmService.test.js dist-test/test/dmTransport.test.js dist-test/test/dmRelayPacer.test.js dist-test/test/dmIncomingBurst.test.js dist-test/test/callScope.test.js dist-test/test/cameraPublication.test.js dist-test/test/remoteMediaSink.test.js dist-test/test/voiceActivity.test.js dist-test/test/dmWire.test.js dist-test/test/dmFriends.test.js test/ipcContract.test.cjs test/ipcRegistry.test.cjs test/appIpc.test.cjs test/dmIpc.test.cjs dist-test/test/rewatchPolicy.test.js dist-test/test/stageTiles.test.js test/motionTokens.test.cjs dist-test/test/nativeFormAnswers.test.js dist-test/test/shellEvents.test.js dist-test/test/initial.test.js",
"test": "tsc -p tsconfig.test.json && node dist-test/test/utils.test.js && node --test dist-test/test/eventBus.test.js dist-test/test/featureAvailability.test.js dist-test/test/botInteractions.test.js dist-test/test/botScreens.test.js dist-test/test/commandUi.test.js dist-test/test/commandAutocomplete.test.js dist-test/test/soundDownloads.test.js dist-test/test/soundDownloadConfirmation.test.js dist-test/test/presence.test.js dist-test/test/gameShare.test.js dist-test/test/rolePermissions.test.js dist-test/test/networkClient.test.js dist-test/test/serverShutdownNotice.test.js dist-test/test/voiceConnection.test.js dist-test/test/voiceModeReconnect.test.js dist-test/test/webRtcManager.test.js dist-test/test/screenSharing.test.js dist-test/test/screenShareControls.test.js dist-test/test/screenShareFailures.test.js dist-test/test/screenSubscriptions.test.js dist-test/test/webScreenPublisher.test.js dist-test/test/videoDiagnostics.test.js dist-test/test/videoTelemetry.test.js dist-test/test/performanceDiagnostics.test.js dist-test/test/responsivenessMonitor.test.js dist-test/test/shortcuts.test.js dist-test/test/releaseNotes.test.js dist-test/test/audioPreferences.test.js dist-test/test/serverSettings.test.js dist-test/test/voiceRooms.test.js dist-test/test/favorites.test.js dist-test/test/cameraEffects.test.js dist-test/test/screenColorPicker.test.js dist-test/test/serverSettingsStore.test.js dist-test/test/hostedServerLifecycle.test.js dist-test/test/messageEditing.test.js dist-test/test/messageSelection.test.js dist-test/test/markdownFormatting.test.js dist-test/test/serverEventCalendar.test.js dist-test/test/autoEntry.test.js dist-test/test/hostedServers.test.js dist-test/test/serverAddress.test.js dist-test/test/backgroundRouting.test.js dist-test/test/nicknames.test.js dist-test/test/livePulse.test.js dist-test/test/botLocalePreferencesStore.test.js dist-test/test/serverMonitor.test.js dist-test/test/serverAudit.test.js dist-test/test/sessionBinding.test.js dist-test/test/localPermissions.test.js dist-test/test/botDeviceStorage.test.js dist-test/test/localExecutionService.test.js dist-test/test/localTools.test.js dist-test/test/localExecutionRenderer.test.js dist-test/test/localExecutionTasks.test.js dist-test/test/localAudioSender.test.js dist-test/test/worker.test.js dist-test/test/workerClient.test.js dist-test/test/workerProtocol.test.js dist-test/test/mediaFolders.test.js dist-test/test/attachmentDownload.test.js dist-test/test/localPreparationView.test.js test/serverManagerLifecycle.test.cjs test/globalInputHookProcess.test.cjs test/developmentProfile.test.cjs test/updaterCompatibility.test.cjs test/crashRecovery.test.cjs test/serverEventCalendarIpc.test.cjs test/hostingSkills.test.cjs test/fileBatchDownloadIpc.test.cjs test/identityExport.test.cjs test/identityService.test.cjs test/localExecutionIpc.test.cjs test/localToolsTab.test.cjs test/steamPresence.test.cjs test/taskbarCallControls.test.cjs test/screenSharingUi.test.cjs test/soundboardLimiter.test.cjs test/trayManager.test.cjs test/onboardingShots.test.cjs test/ipcSenderPolicy.test.cjs test/windowSandbox.test.cjs test/recentSoundSaveIpc.test.cjs test/webDeployment.test.cjs test/webMiniapps.test.cjs test/zipFormat.test.cjs dist-test/test/settingsPersistence.test.js test/startupLoading.test.cjs dist-test/test/pointerReorder.test.js dist-test/test/rovingFocus.test.js && node --test dist-test/test/authHardening.test.js dist-test/test/hardeningAndOverlay.test.js dist-test/test/dmService.test.js dist-test/test/dmTransport.test.js dist-test/test/dmRelayPacer.test.js dist-test/test/dmIncomingBurst.test.js dist-test/test/callScope.test.js dist-test/test/cameraPublication.test.js dist-test/test/remoteMediaSink.test.js dist-test/test/voiceActivity.test.js dist-test/test/dmWire.test.js dist-test/test/dmFriends.test.js test/ipcContract.test.cjs test/ipcRegistry.test.cjs test/appIpc.test.cjs test/dmIpc.test.cjs dist-test/test/rewatchPolicy.test.js dist-test/test/stageTiles.test.js test/motionTokens.test.cjs dist-test/test/nativeFormAnswers.test.js dist-test/test/shellEvents.test.js dist-test/test/initial.test.js",
"test:development": "node test/developmentProfileSmoke.cjs",
"test:categories": "node --test test/channelCategoriesDom.test.cjs",
"test:calendar": "tsc -p tsconfig.test.json && node --test dist-test/test/serverEventCalendar.test.js dist-test/test/calendarDate.test.js test/serverEventCalendarIpc.test.cjs test/recentSoundSaveIpc.test.cjs",
Expand Down
26 changes: 25 additions & 1 deletion apps/client/scripts/buildMiniapps.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,34 @@ export const WEB_DEV_PORT = 5174;
export const MINIAPPS_DEV_URL = 'http://localhost:5175/';

/** Runs inside the miniapp frame before the document replaces the page. */
function loader(origins, messages) {
export function loader(origins, messages) {
const token = location.hash.slice(1);
const host = window.parent;
if (host === window || !/^[0-9a-f-]{36}$/.test(token)) return;
// This page is HTTPS: the browser blocks the miniapp's plain http/ws calls (mixed content) beyond the
// user's own machine. The calls go on unchanged; the app only learns which service was refused, once.
const reported = new Set();
const watch = (value) => {
try {
const url = new URL(String(value instanceof Request ? value.url : value), location.href);
if ((url.protocol !== 'http:' && url.protocol !== 'ws:')
|| ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname) || reported.has(url.origin)) return;
reported.add(url.origin);
host.postMessage({ type: messages.insecure, token, origin: url.origin }, '*');
} catch { /* Not a URL: the call fails on its own. */ }
};
const fetch = window.fetch;
window.fetch = function (input, init) { watch(input); return fetch.call(this, input, init); };
const open = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (method, url, ...rest) { watch(url); return open.call(this, method, url, ...rest); };
for (const name of ['WebSocket', 'EventSource']) {
const Native = window[name];
if (typeof Native !== 'function') continue;
const Watched = function (url, ...rest) { watch(url); return new Native(url, ...rest); };
Watched.prototype = Native.prototype;
Object.setPrototypeOf(Watched, Native);
window[name] = Watched;
}
const receive = (event) => {
const data = event.data;
if (event.source !== host || !origins.includes(event.origin) || !data || data.type !== messages.document ||
Expand Down
20 changes: 18 additions & 2 deletions apps/client/src/renderer/core/webrtc/BrowserScreenSubscription.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,22 @@ export class BrowserScreenSubscription {
this.options.onError(error);
}

/**
* The first frame never came. Video that arrived yet never decoded is the browser's decoder refusing the
* stream (Firefox accepts AV1 in its capability probe and then decodes none of a screen), not the network.
*/
private async startupExpired(message: string): Promise<void> {
let undecoded = false;
try {
(await this.stats())?.forEach(report => {
if (report.type === 'inbound-rtp' && report.kind === 'video' && report.packetsReceived > 0 && !report.framesDecoded)
undecoded = true;
});
} catch { /* The stats only refine the reason. */ }
this.fail(undecoded ? new BrowserScreenCodecError(`${message} Its video arrived, but this browser decoded none of it.`)
: new Error(message));
}

public start(): Promise<void> {
if (this.starting) return this.starting;
if (this.stopping) return Promise.reject(new DOMException('Screen Watch was retired.', 'AbortError'));
Expand All @@ -77,7 +93,7 @@ export class BrowserScreenSubscription {
}
if (this.stopping) throw new DOMException('Screen Watch was retired during codec discovery.', 'AbortError');
this.started = true;
this.startupTimer = setTimeout(() => this.fail(new Error('The browser screen did not present its first frame.')), 30000);
this.startupTimer = setTimeout(() => { void this.startupExpired('The browser screen did not present its first frame.'); }, 30000);
await this.options.send(this.envelope({ action: 'watch', backend: 'browser', quality: this.options.quality }));
})();
return this.starting;
Expand Down Expand Up @@ -139,7 +155,7 @@ export class BrowserScreenSubscription {
else if (signal.capture.mode === 'game' && !this.isPlaying && !this.gameStartupWait) {
this.gameStartupWait = true;
if (this.startupTimer) clearTimeout(this.startupTimer);
this.startupTimer = setTimeout(() => this.fail(new Error('Game Capture and its Normal fallback did not produce a frame.')),
this.startupTimer = setTimeout(() => { void this.startupExpired('Game Capture and its Normal fallback did not produce a frame.'); },
NATIVE_SCREEN_GAME_STARTUP_TIMEOUT_MS);
}
return;
Expand Down
Loading
Loading