diff --git a/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts b/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts index b90b3f38..4eaf6cf2 100644 --- a/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts +++ b/apps/client/src/renderer/core/webrtc/BrowserScreenSfu.ts @@ -5,7 +5,7 @@ import { type NativeScreenVideoProfile, type SfuConsumedPayload, type SfuProducerClosedPayload, type SfuProducersListPayload, type SfuRouterRtpCapabilitiesPayload, type SfuWebRtcTransportCreatedPayload, } from '@monky/shared'; -import { BrowserScreenCodecError, av1ReceiveLevel, h264MaxReceiveLevel, supportsBrowserAv1Level } from './browserScreenCodecs'; +import { BrowserScreenCodecError, av1ReceiveLevel, browserReceivesH264Main, h264MaxReceiveLevel, supportsBrowserAv1Level } from './browserScreenCodecs'; export type BrowserScreenRpc = (method: NativeScreenRpcMethod, payload: Record) => Promise; type RouterResponse = Omit & { rtpCapabilities: SfuTypes.RtpCapabilities }; @@ -169,10 +169,15 @@ export class BrowserScreenSfu { ? h264MaxReceiveLevel(codec.parameters?.['profile-level-id'], this.options.profile) : null; if (maxReceiveLevel) codec.parameters = { ...codec.parameters, 'max-recv-level': maxReceiveLevel }; } + // A screen nobody here can decode is refused when consuming: by the server (no common profile) or by + // the browser's own description. Without an H.264 Main receiver that is the codec, not the network. + const codecRefusal = (error: unknown): unknown => producer.kind === 'video' && !browserReceivesH264Main() + && !(error instanceof DOMException) + ? new BrowserScreenCodecError(`This browser cannot consume the ${selectedMime} screen: ${String(error)}`) : error; const consumed = await rpc(MessageType.SFU_CONSUME, { channelId: call.channelId, transportId: transport.id, producerId: producer.producerId, rtpCapabilities: capabilities, - }); + }).catch((error: unknown) => { throw codecRefusal(error); }); if (!('id' in consumed)) { if (consumed.channelId !== call.channelId || consumed.producerId !== producer.producerId) throw new Error('Screen consumption ended another producer.'); @@ -205,7 +210,7 @@ export class BrowserScreenSfu { id: consumed.id, producerId: consumed.producerId, kind: consumed.kind, rtpParameters: consumed.rtpParameters, appData: metadata.appData, ...(consumed.kind === 'audio' ? { codecOptions: { opusStereo: true } } : {}), - }); + }).catch((error: unknown) => { throw codecRefusal(error); }); if (!selected()) { consumer.close(); await this.closeConsumer(consumed.id); return; } entry.consumer = consumer; consumer.on('trackended', () => { diff --git a/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts b/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts index 7b10926a..2170f130 100644 --- a/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts +++ b/apps/client/src/renderer/core/webrtc/browserScreenCodecs.ts @@ -43,14 +43,38 @@ export function av1ReceiveLevel(parameters: Readonly>, v return av1 !== null && av1.level < minimum ? minimum : null; } -/** Whether this browser's WebRTC receiver lists the codec of Monky screens at all: H.264 Main or AV1 profile 0. */ +const h264Receiver = (codec: { mimeType: string; sdpFmtpLine?: string }, profile: RegExp): boolean => + codec.mimeType.toLowerCase() === 'video/h264' + && profile.test(/(?:^|;)\s*profile-level-id=([0-9a-f]{6})(?:;|$)/i.exec(codec.sdpFmtpLine ?? '')?.[1] ?? '') + && /(?:^|;)\s*packetization-mode=1(?:;|$)/.test(codec.sdpFmtpLine ?? ''); + +/** Whether this browser receives H.264 Main, what the desktop app and the web's better format send. */ +export function browserReceivesH264Main(): boolean { + return RTCRtpReceiver.getCapabilities('video')?.codecs.some(codec => h264Receiver(codec, /^4d/i)) ?? false; +} + +/** + * Whether this browser's WebRTC receiver lists the codec of Monky screens at all: AV1 profile 0, or H.264 as + * Main or as the Constrained Baseline the web shares by default (Firefox and Safari list only the latter). + */ export function browserReceivesScreenCodec(selected: ScreenCodec = 'h264'): boolean { return RTCRtpReceiver.getCapabilities('video')?.codecs.some(codec => selected === 'av1' ? codec.mimeType.toLowerCase() === 'video/av1' && av1Parameters(fmtpParameters(codec.sdpFmtpLine ?? ''))?.profile === 0 - : codec.mimeType.toLowerCase() === 'video/h264' - && /(?:^|;)\s*profile-level-id=4d[0-9a-f]{4}(?:;|$)/i.test(codec.sdpFmtpLine ?? '') - && /(?:^|;)\s*packetization-mode=1(?:;|$)/.test(codec.sdpFmtpLine ?? '')) ?? false; + : h264Receiver(codec, /^(4d|42e0)/i)) ?? false; +} + +/** + * What to tell a browser about a screen in `codec` it could not decode: `main`, an H.264 Main share it lacks + * (Firefox, Safari); `switch`, another codec the publisher can change to the H.264 it receives; `codec`, a format + * it does not decode with nothing to switch to (Firefox and AV1); `null`, nothing codec-specific. + */ +export function browserScreenRefusal(codec: ScreenCodec): 'main' | 'switch' | 'codec' | null { + const main = browserReceivesH264Main(); + if (codec === 'h264' && !main && browserReceivesScreenCodec('h264')) return 'main'; + if (codec !== 'h264' && main) return 'switch'; + if (!browserReceivesScreenCodec(codec) || !main) return 'codec'; + return null; } export async function supportsBrowserScreenCodec( @@ -58,11 +82,14 @@ export async function supportsBrowserScreenCodec( ): Promise { // For AV1 the advertised level is only a default: the decoder probe below decides. if (!browserReceivesScreenCodec(selected) || !navigator.mediaCapabilities?.decodingInfo) return false; + // A browser without Main is probed with the Constrained Baseline it receives, at the same level. + const h264 = getScreenH264ProfileLevelId(profile); + const profileLevelId = browserReceivesH264Main() ? h264 : `42e0${h264.slice(4)}`; const capability = await navigator.mediaCapabilities.decodingInfo({ type: 'webrtc', video: { contentType: selected === 'av1' ? `video/AV1;profile=0;level-idx=${getScreenAv1MinimumLevelIndex(profile)};tier=0` - : `video/H264;profile-level-id=${getScreenH264ProfileLevelId(profile)};packetization-mode=1`, + : `video/H264;profile-level-id=${profileLevelId};packetization-mode=1`, width: profile.width, height: profile.height, framerate: profile.fps, bitrate: profile.maxBitrateKbps * 1000, }, }); @@ -73,6 +100,9 @@ export function assertBrowserScreenCodec( sdp: string, selected: ScreenCodec, profile?: Readonly, ): void { const videoSections = sdp.split(/(?=^m=)/m).filter(section => section.startsWith('m=video ')); + // A receiver that shares no codec with the screen answers its video rejected (port 0): that is the codec. + if (videoSections.length && videoSections.every(section => section.split(' ')[1] === '0')) + throw new BrowserScreenCodecError(`This browser receives no ${selected.toUpperCase()} profile the screen offers.`); for (const section of videoSections) { const [media, ...lines] = section.split(/\r?\n/); if (media.split(' ')[1] === '0') continue; diff --git a/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts b/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts index b0fc0d6c..c1396fbf 100644 --- a/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts +++ b/apps/client/src/renderer/core/webrtc/nativeScreen/LocalScreenSources.ts @@ -8,6 +8,7 @@ import { emitOutsideRouting } from '../../sessionRouting'; import { t } from '../../../i18n'; import { videoService, type NativeScreenCapture } from '../../VideoService'; import { settingsStore } from '../../../stores/settingsStore'; +import { platformSupports } from '../../../platform/capabilities'; import { customVideoFpsLimit } from '../../../utils/qualityProfileLimits'; import { acceptScreenEncoding } from '../../screenEncoding'; import { @@ -17,6 +18,9 @@ import { /** Why viewers see a frozen picture of a local source: its window is minimized or hidden, or a browser stopped drawing it. */ export type NativeScreenSourceWarning = 'hidden' | 'covered'; +/** The format a browser publisher encodes in (#858); the desktop chooses its codec through its own settings. */ +const browserFormat = () => platformSupports('browserScreenPicker') ? { browserFormat: settingsStore.browserScreenFormat } : {}; + export type SourceInput = Omit & { shareId: string; video: NativeScreenVideoProfile; audio: boolean; replacesAudioShareIds?: readonly string[]; @@ -151,7 +155,7 @@ export class LocalScreenSources { desktopSourceId: input.desktopSourceId, video: input.video, audio: input.audio, audioBitrateKbps: input.audioBitrateKbps, preserveAspectRatio: input.preserveAspectRatio ?? true, encodingMode: settingsStore.screenEncodingMode, codec: settingsStore.preferredScreenCodec, - encodingStrategy: settingsStore.screenEncodingStrategy, + encodingStrategy: settingsStore.screenEncodingStrategy, ...browserFormat(), ...(input.replacesAudioShareIds?.length ? { replacesAudioShareIds: [...input.replacesAudioShareIds] } : {}), ...(input.captureKind ? { captureKind: input.captureKind } : {}), }).then(result => { @@ -221,7 +225,7 @@ export class LocalScreenSources { preserveAspectRatio: capture.preserveAspectRatio ?? true, video, audio, audioBitrateKbps, encodingMode: settingsStore.screenEncodingMode, codec: settingsStore.preferredScreenCodec, - encodingStrategy: settingsStore.screenEncodingStrategy, + encodingStrategy: settingsStore.screenEncodingStrategy, ...browserFormat(), }); this.calls.current(call); if (!isCurrentCapture() || previous.removing || call.sources.get(shareId) !== previous) { diff --git a/apps/client/src/renderer/i18n/locales/en.ts b/apps/client/src/renderer/i18n/locales/en.ts index 9bd49c8d..afa69f28 100644 --- a/apps/client/src/renderer/i18n/locales/en.ts +++ b/apps/client/src/renderer/i18n/locales/en.ts @@ -959,9 +959,15 @@ export const en: TranslationMap = { 'screenShare.chromiumReceiverSettingsHint': 'You are using the Chromium receiver, which cannot receive AV1 at high-resolution profiles. Select Native in Settings → Quality & sharing → Screen reception and try watching again, or ask the publisher to use H.264.', 'screenShare.browserReceiverHint': 'Ask the publisher to use H.264 and try watching again.', 'screenShare.browserCannotReceive': 'This browser cannot receive {codec} screens, the format of this share. Watch it in Chrome, Edge or the desktop app.', + 'screenShare.browserCannotReceiveMain': 'This browser cannot receive this share: it uses H.264 Main, from the desktop app or the web\'s better quality option. Watch it in Chrome, Edge or the desktop app.', + 'screenShare.browserFormat': 'Video format', + 'screenShare.browserFormatCompatible': 'Compatible', + 'screenShare.browserFormatCompatibleDescription': 'H.264 everyone can watch: Chrome, Edge, Firefox, Safari and the desktop app. Recommended.', + 'screenShare.browserFormatMain': 'Better quality', + 'screenShare.browserFormatMainDescription': 'H.264 Main: a better picture with the same bandwidth. Firefox and Safari cannot watch it.', 'desktopOnly.badge': 'Desktop', 'desktopOnly.tooltip': 'Available in the desktop app', - 'desktopOnly.reason.screenShare': 'This browser cannot share screens in Monky\'s format (H.264). Use Chrome or Edge, or the desktop app.', + 'desktopOnly.reason.screenShare': 'This browser cannot share screens to Monky. Use Chrome, Edge or Firefox, or the desktop app.', 'desktopOnly.title': 'Available in the desktop app', 'desktopOnly.message': '{feature}: this feature only works in the Monky desktop app. Download the app to use it.', 'desktopOnly.panelNotice': '{feature}: available only in the Monky desktop app. Here you can only look at the options.', diff --git a/apps/client/src/renderer/i18n/locales/pt-BR.ts b/apps/client/src/renderer/i18n/locales/pt-BR.ts index 1dafb827..95a5b8d1 100644 --- a/apps/client/src/renderer/i18n/locales/pt-BR.ts +++ b/apps/client/src/renderer/i18n/locales/pt-BR.ts @@ -921,9 +921,15 @@ export const ptBR = { 'screenShare.chromiumReceiverSettingsHint': 'Você está usando o receptor Chromium, que não recebe AV1 em perfis de alta resolução. Selecione Nativo em Configurações → Qualidade e compartilhamento → Recepção de tela e tente assistir novamente, ou peça ao transmissor para usar H.264.', 'screenShare.browserReceiverHint': 'Peça a quem está transmitindo para usar H.264 e tente assistir novamente.', 'screenShare.browserCannotReceive': 'Este navegador não recebe telas em {codec}, o formato desta transmissão. Assista pelo Chrome, pelo Edge ou pelo app desktop.', + 'screenShare.browserCannotReceiveMain': 'Este navegador não recebe esta transmissão: ela usa H.264 Main, do app desktop ou da opção de melhor qualidade da web. Assista pelo Chrome, pelo Edge ou pelo app desktop.', + 'screenShare.browserFormat': 'Formato do vídeo', + 'screenShare.browserFormatCompatible': 'Compatível', + 'screenShare.browserFormatCompatibleDescription': 'H.264 que todos assistem: Chrome, Edge, Firefox, Safari e o app desktop. Recomendado.', + 'screenShare.browserFormatMain': 'Melhor qualidade', + 'screenShare.browserFormatMainDescription': 'H.264 Main: imagem melhor com a mesma banda. O Firefox e o Safari não conseguem assistir.', 'desktopOnly.badge': 'Desktop', 'desktopOnly.tooltip': 'Disponível no app desktop', - 'desktopOnly.reason.screenShare': 'Este navegador não transmite tela no formato do Monky (H.264). Use o Chrome ou o Edge, ou o app desktop.', + 'desktopOnly.reason.screenShare': 'Este navegador não consegue transmitir tela para o Monky. Use o Chrome, o Edge ou o Firefox, ou o app desktop.', 'desktopOnly.title': 'Disponível no app desktop', 'desktopOnly.message': '{feature}: este recurso funciona só no app do Monky para computador. Baixe o app para usar.', 'desktopOnly.panelNotice': '{feature}: disponível só no app do Monky para computador. Aqui você só pode ver as opções.', diff --git a/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts b/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts index dfbdb4cb..48283b49 100644 --- a/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts +++ b/apps/client/src/renderer/platform/web/webScreen/browserScreenPeer.ts @@ -1,10 +1,12 @@ -import type { NativeScreenCall } from '@monky/shared'; +import type { BrowserScreenFormat, NativeScreenCall } from '@monky/shared'; import { screenSenderCodecs } from './browserScreenProfile'; import { sentMediaBinding, withScreenStreamId, type ScreenMediaBinding } from './screenSdp'; import type { ScreenIceCandidate, ScreenMediaKind, ScreenPeerPort } from './screenP2pPublisherPeer'; export interface BrowserScreenPeerOptions { readonly iceServers: NativeScreenCall['iceServers']; + /** The only video codecs this connection may send (#858). */ + readonly format: BrowserScreenFormat; /** The screen's instance: the stream id viewers group its video and sound by. */ readonly streamId: string; readonly video: MediaStreamTrack; @@ -33,10 +35,10 @@ export class BrowserScreenPeer implements ScreenPeerPort { direction: 'sendonly', streams: [this.stream], sendEncodings: [{ active: false, maxBitrate: options.maxBitrateBps, maxFramerate: options.maxFramerate }], }); - const codecs = screenSenderCodecs(RTCRtpSender.getCapabilities('video')?.codecs ?? []); + const codecs = screenSenderCodecs(RTCRtpSender.getCapabilities('video')?.codecs ?? [], options.format); if (!codecs.length) { this.peer.close(); - throw new Error('This browser cannot send the H.264 Main video screen viewers receive.'); + throw new Error(`This browser cannot send screens in the ${options.format} format.`); } video.setCodecPreferences(codecs); this.transceivers.set('video', video); diff --git a/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts b/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts index 43d78cae..7093535a 100644 --- a/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts +++ b/apps/client/src/renderer/platform/web/webScreen/browserScreenProfile.ts @@ -1,4 +1,7 @@ -import { getScreenShareProfile, nativeScreenVideoProfileSchema, type NativeScreenVideoProfile } from '@monky/shared'; +import { + browserScreenFormatSchema, getScreenShareProfile, nativeScreenVideoProfileSchema, + type BrowserScreenFormat, type NativeScreenVideoProfile, +} from '@monky/shared'; /** What a captured video track reports of itself (`MediaStreamTrack.getSettings()`). */ export interface CapturedVideoSettings { @@ -38,18 +41,27 @@ export interface CodecCapability { const fmtp = (codec: CodecCapability, key: string): string | undefined => codec.sdpFmtpLine?.split(';').map(part => part.trim().split('=')).find(([name]) => name === key)?.[1]; -/** H.264 Main in non-interleaved mode: what every screen receiver, native or browser, decodes. */ -export function isScreenH264(codec: CodecCapability): boolean { - return codec.mimeType.toLowerCase() === 'video/h264' && /^4d[0-9a-f]{4}$/i.test(fmtp(codec, 'profile-level-id') ?? '') +/** + * Whether a sender codec encodes a browser format (#858): H.264 in non-interleaved mode, as Constrained + * Baseline (42e0..) for `compatible`, which every receiver decodes, or as Main (4d..) for `main`. + */ +export function isScreenCodec(codec: CodecCapability, format: BrowserScreenFormat): boolean { + const profile = format === 'main' ? /^4d[0-9a-f]{4}$/i : /^42e0[0-9a-f]{2}$/i; + return codec.mimeType.toLowerCase() === 'video/h264' && profile.test(fmtp(codec, 'profile-level-id') ?? '') && fmtp(codec, 'packetization-mode') === '1'; } /** - * The video codecs a browser screen sender offers, in order: H.264 Main and its retransmission. Viewers - * accept nothing else for a source announced as H.264, so an empty list means this browser cannot share. + * The video codecs a browser screen sender offers for `format`, in order: the format's codec and its + * retransmission. An empty list means this browser cannot share in that format. */ -export function screenSenderCodecs(codecs: readonly T[]): T[] { - const h264 = codecs.filter(isScreenH264); - if (!h264.length) return []; - return [...h264, ...codecs.filter(codec => codec.mimeType.toLowerCase() === 'video/rtx')]; +export function screenSenderCodecs(codecs: readonly T[], format: BrowserScreenFormat): T[] { + const matching = codecs.filter(codec => isScreenCodec(codec, format)); + if (!matching.length) return []; + return [...matching, ...codecs.filter(codec => codec.mimeType.toLowerCase() === 'video/rtx')]; +} + +/** The formats this browser can share in, the compatible one first. */ +export function browserScreenFormats(codecs: readonly CodecCapability[]): BrowserScreenFormat[] { + return browserScreenFormatSchema.options.filter(format => screenSenderCodecs(codecs, format).length > 0); } diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts index 838b596c..13a7db00 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenPublisher.ts @@ -1,7 +1,7 @@ import { getScreenShareProfile, nativeScreenFailureReason, nativeScreenSignalSchema, screenShareProfileKey, type NativeScreenCall, type NativeScreenFailure, type NativeScreenSignalPayload, type NativeScreenSource, - type NativeScreenVideoProfile, type ScreenShareQuality, + type NativeScreenVideoProfile, type ScreenShareQuality, type BrowserScreenFormat, } from '@monky/shared'; import { BrowserScreenPeer } from './browserScreenPeer'; import type { DisplayCapture } from './displayCaptures'; @@ -48,6 +48,8 @@ const capacity = (): Error => Object.assign(new Error('The screen profile has re export interface WebScreenPublisherOptions { readonly call: NativeScreenCall; readonly source: NativeScreenSource; + /** How this screen is encoded; its codec is the source's. */ + readonly format: BrowserScreenFormat; readonly capture: DisplayCapture; readonly audioBitrateKbps: number; readonly signal: (signal: NativeScreenSignalPayload) => Promise; @@ -160,7 +162,7 @@ export class WebScreenPublisher { const { call, source, audioBitrateKbps } = this.options; let peer: ScreenP2pPublisherPeer | null = null; const port = new BrowserScreenPeer({ - iceServers: call.iceServers, streamId: source.instanceId, video: pipeline.video, audio: pipeline.audio, + iceServers: call.iceServers, format: this.options.format, streamId: source.instanceId, video: pipeline.video, audio: pipeline.audio, maxBitrateBps: pipeline.profile.maxBitrateKbps * 1000, maxFramerate: pipeline.profile.fps, audioBitrateBps: audioBitrateKbps * 1000, onCandidate: candidate => peer?.localCandidate(candidate), @@ -186,7 +188,7 @@ export class WebScreenPublisher { private async pipelineFor(viewer: Viewer): Promise { const { source, capture, call } = this.options; - const profile = getScreenShareProfile(source.video, viewer.quality, 'h264'); + const profile = getScreenShareProfile(source.video, viewer.quality, source.codec); const key = screenShareProfileKey(profile); const existing = this.pipelines.get(key); if (existing?.closing) { @@ -200,7 +202,7 @@ export class WebScreenPublisher { pipeline.viewers.set(viewer.sessionId, viewer); if (!existing && call.mode === 'sfu') { const sfu = new WebScreenSfuRendition({ - channelId: call.channelId, shareId: source.shareId, + channelId: call.channelId, shareId: source.shareId, format: this.options.format, rendition: { sourceInstanceId: source.instanceId, pipelineId: pipeline.id, video: profile }, video: pipeline.video, audio: pipeline.audio, audioBitrateBps: this.options.audioBitrateKbps * 1000, rpc: this.options.rpc, onError: error => { void this.track(this.failPipeline(pipeline, error)); }, diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts index 21d24794..32b61d3b 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenRuntime.ts @@ -5,7 +5,7 @@ import { type NativeScreenParticipant, type NativeScreenPresentation, type NativeScreenSignalPayload, type NativeScreenSource, } from '@monky/shared'; import type { PlatformApi } from '../../api'; -import { browserScreenProfile, screenSenderCodecs } from './browserScreenProfile'; +import { browserScreenFormats, browserScreenProfile } from './browserScreenProfile'; import { DisplayCaptures, type DisplayCapture } from './displayCaptures'; import { WebScreenBridge } from './webScreenBridge'; import { WebScreenPublisher } from './webScreenPublisher'; @@ -45,14 +45,13 @@ const errorText = (error: unknown): string => (error instanceof Error ? error.me /** What a browser tab can do with screens: capture through its own dialog and send; receiving is the browser receiver's. */ export function webScreenCapabilities(): NativeScreenCapabilities { const display = typeof navigator !== 'undefined' && typeof navigator.mediaDevices?.getDisplayMedia === 'function'; - const encoder = typeof RTCRtpSender !== 'undefined' - && screenSenderCodecs(RTCRtpSender.getCapabilities('video')?.codecs ?? []).length > 0; - const capture = display && encoder; + const formats = typeof RTCRtpSender === 'undefined' ? [] : browserScreenFormats(RTCRtpSender.getCapabilities('video')?.codecs ?? []); + const capture = display && formats.length > 0; // Only Chromium shares a tab's or the system's sound, and only Chromium lists this constraint. const captureAudio = capture && 'suppressLocalAudioPlayback' in navigator.mediaDevices.getSupportedConstraints(); return { capture, captureAudio, receive: false, captureKinds: capture ? ['window', 'monitor'] : [], backend: null, - reason: !display ? 'platform' : !encoder ? 'encoder' : null, + reason: !display ? 'platform' : !formats.length ? 'encoder' : null, browserFormats: capture ? formats : [], }; } @@ -242,6 +241,9 @@ export class WebScreenRuntime { throw new Error('The capture kind does not match what the browser captured.'); if (command.audio && (!capture.audioTrack || !capabilities.captureAudio)) throw new Error('The browser did not share sound with this capture.'); + // The compatible format unless the publisher chose a better one this browser encodes; both are H.264. + const format = command.browserFormat ?? 'compatible'; + if (!capabilities.browserFormats?.includes(format)) throw new Error(`This browser cannot share in the ${format} format.`); call.admitting.add(command.shareId); try { if (replacement) await this.retireSource(call, replacement); @@ -252,7 +254,7 @@ export class WebScreenRuntime { video: browserScreenProfile(command.video, capture.video.getSettings()), audio: command.audio, codec: 'h264', }); const publisher = new WebScreenPublisher({ - call: call.config, source, capture, audioBitrateKbps: command.audioBitrateKbps, + call: call.config, source, format, capture, audioBitrateKbps: command.audioBitrateKbps, signal: signal => this.bridge.signal(call.config.callId, signal), rpc: (method, payload) => this.bridge.rpc(call.config.callId, call.config.channelId, method, payload), onError: error => console.warn('[WebScreen] A screen viewer failed:', error), diff --git a/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts b/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts index 024e6957..8da00b98 100644 --- a/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts +++ b/apps/client/src/renderer/platform/web/webScreen/webScreenSfuRendition.ts @@ -1,12 +1,13 @@ import { Device, type types as SfuTypes } from 'mediasoup-client'; -import { MessageType, type NativeScreenRendition } from '@monky/shared'; -import { isScreenH264 } from './browserScreenProfile'; +import { MessageType, type BrowserScreenFormat, type NativeScreenRendition } from '@monky/shared'; +import { isScreenCodec } from './browserScreenProfile'; export type WebScreenRpc = (method: MessageType, payload: Record) => Promise; export interface WebScreenSfuRenditionOptions { readonly channelId: string; readonly shareId: string; + readonly format: BrowserScreenFormat; readonly rendition: NativeScreenRendition; readonly video: MediaStreamTrack; readonly audio: MediaStreamTrack | null; @@ -59,11 +60,12 @@ export class WebScreenSfuRendition { throw new Error('The screen router capabilities belong to another channel.'); await this.device.load({ routerRtpCapabilities: router.rtpCapabilities }); this.current(); - const codec = this.device.rtpCapabilities.codecs?.find(value => isScreenH264({ + const { format } = this.options; + const codec = this.device.rtpCapabilities.codecs?.find(value => value.kind === 'video' && isScreenCodec({ mimeType: value.mimeType, sdpFmtpLine: Object.entries(value.parameters ?? {}).map(([key, parameter]) => `${key}=${String(parameter)}`).join(';'), - })); - if (!codec || !this.device.canProduce('video')) throw new Error('The server and this browser share no H.264 Main screen codec.'); + }, format)); + if (!codec || !this.device.canProduce('video')) throw new Error(`The server and this browser share no ${format} screen codec.`); const created = await rpc(MessageType.SFU_CREATE_WEBRTC_TRANSPORT, { channelId, direction: 'send', purpose: 'screen', screenSessionId: rendition.pipelineId, }); diff --git a/apps/client/src/renderer/stores/settingsStore.ts b/apps/client/src/renderer/stores/settingsStore.ts index 06852bc8..fb83c0e9 100644 --- a/apps/client/src/renderer/stores/settingsStore.ts +++ b/apps/client/src/renderer/stores/settingsStore.ts @@ -16,7 +16,7 @@ import { type BotFormValues, type BotLocale, type ScreenEncodingMode, - type ScreenCodec, type ScreenEncodingStrategy, + type ScreenCodec, type ScreenEncodingStrategy, type BrowserScreenFormat, browserScreenFormatSchema, } from '@monky/shared'; import { appEvents } from '../core/EventBus'; import { @@ -53,6 +53,7 @@ function savedSettings(store: SettingsStore) { screenEncodingMode: store.screenEncodingMode, screenEncodingStrategy: store.screenEncodingStrategy, preferredScreenCodec: store.preferredScreenCodec, + browserScreenFormat: store.browserScreenFormat, vadSensitivity: store.vadSensitivity, inputVolume: store.inputVolume, outputVolume: store.outputVolume, @@ -204,6 +205,8 @@ export class SettingsStore { public screenEncodingMode: ScreenEncodingMode = 'hardware'; public screenEncodingStrategy: ScreenEncodingStrategy = 'automatic'; public preferredScreenCodec: ScreenCodec = 'h264'; + /** How the web version encodes the screens it shares (#858): compatible with every viewer by default. */ + public browserScreenFormat: BrowserScreenFormat = 'compatible'; public customProfile: QualityProfile = { ...DEFAULT_CUSTOM_PROFILE }; public inputMode: 'voice_activity' | 'push_to_talk' = 'voice_activity'; // #186 public pttKey: PttKeyBinding = { code: 'KeyV', display: 'V', keyType: 'keyboard', keyCode: 47 }; @@ -327,6 +330,7 @@ export class SettingsStore { this.screenEncodingMode = 'hardware'; this.screenEncodingStrategy = 'automatic'; this.preferredScreenCodec = 'h264'; + this.browserScreenFormat = 'compatible'; this.screenShareReceiver = 'native'; this.soundboardLimiterEnabled = false; this.soundboardLoudnessLimit = 6; @@ -351,6 +355,8 @@ export class SettingsStore { this.qualityPreset = restoreQualityPreset(parsed.qualityPreset); this.screenEncodingMode = parsed.screenEncodingMode === 'software' ? 'software' : 'hardware'; this.preferredScreenCodec = parsed.preferredScreenCodec === 'av1' ? 'av1' : 'h264'; + const browserFormat = browserScreenFormatSchema.safeParse(parsed.browserScreenFormat); + this.browserScreenFormat = browserFormat.success ? browserFormat.data : 'compatible'; this.screenEncodingStrategy = parsed.screenEncodingStrategy === 'manual' ? 'manual' : parsed.screenEncodingStrategy === undefined && (parsed.screenEncodingMode === 'software' || parsed.preferredScreenCodec === 'h264' @@ -915,6 +921,11 @@ export class SettingsStore { catch (error) { this.screenShareReceiver = previous; throw error; } } + public setBrowserScreenFormat(format: BrowserScreenFormat): void { + this.browserScreenFormat = browserScreenFormatSchema.parse(format); + this.save(); + } + private limitCustomProfile(): void { const normalized = normalizeCustomQualityProfile(this.customProfile); if (JSON.stringify(normalized) !== JSON.stringify(this.customProfile)) { diff --git a/apps/client/src/renderer/views/VoiceStageView.ts b/apps/client/src/renderer/views/VoiceStageView.ts index 18bcfa7b..fa630164 100644 --- a/apps/client/src/renderer/views/VoiceStageView.ts +++ b/apps/client/src/renderer/views/VoiceStageView.ts @@ -14,7 +14,7 @@ import { settingsStore } from '../stores/settingsStore'; import { voiceStore, VoiceStore } from '../stores/voiceStore'; import { videoService } from '../core/VideoService'; import { webRtcManager } from '../core/WebRtcManager'; -import { browserReceivesScreenCodec } from '../core/webrtc/browserScreenCodecs'; +import { browserScreenRefusal } from '../core/webrtc/browserScreenCodecs'; import type { NativeScreenWatchState } from '../core/webrtc/NativeScreenController'; import { getAvatarUrl } from '../utils/avatar'; import { screenShareFailureMessage } from '../utils/screenShareFailure'; @@ -1226,12 +1226,12 @@ export class VoiceStageView { // Choosing the native receiver exists only where the platform has one (the desktop). if (platformSupports('nativeScreenAudio')) return `${failure} ${t('screenShare.chromiumReceiverSettingsHint')}`; const codec = webRtcManager.getNativeScreenSource(sessionId, shareId)?.codec ?? 'h264'; - // The publisher can help only a browser that receives H.264, by switching another codec to it. - if (codec !== 'h264' && browserReceivesScreenCodec('h264')) return `${failure} ${t('screenShare.browserReceiverHint')}`; - // Without the share's codec, or with nothing to switch to (Firefox and Safari have no H.264 Main receiver). - if (!browserReceivesScreenCodec(codec) || !browserReceivesScreenCodec('h264')) - return t('screenShare.browserCannotReceive', { codec: codec === 'av1' ? 'AV1' : 'H.264' }); - return failure; + switch (browserScreenRefusal(codec)) { + case 'main': return t('screenShare.browserCannotReceiveMain'); + case 'switch': return `${failure} ${t('screenShare.browserReceiverHint')}`; + case 'codec': return t('screenShare.browserCannotReceive', { codec: codec === 'av1' ? 'AV1' : 'H.264' }); + default: return failure; + } } private refreshNativeScreenState(): void { diff --git a/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts b/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts index 21fcbd9a..a57f3c21 100644 --- a/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts +++ b/apps/client/src/renderer/views/screenShare/BrowserScreenSharePicker.ts @@ -1,4 +1,4 @@ -import { PUBLIC_AUDIENCE, type ResourceAudience } from '@monky/shared'; +import { PUBLIC_AUDIENCE, browserScreenFormatSchema, type BrowserScreenFormat, type ResourceAudience } from '@monky/shared'; import { html, trustedHtml } from '../../utils/html'; import { appEvents } from '../../core/EventBus'; import { captureScreenShareCall, screenShareLimit, type ScreenShareCall } from '../../core/screenShareControls'; @@ -9,7 +9,8 @@ import { nativeScreenProfile } from '../../core/webrtc/NativeScreenController'; import { voiceStore } from '../../stores/voiceStore'; import { setButtonLoading } from '../../utils/buttonLoading'; import { openModal, type ModalShell } from '../../utils/modalShell'; -import { renderSwitch } from '../../utils/formControls'; +import { renderChoiceCards, renderSwitch, selectChoiceCard, type ChoiceCard } from '../../utils/formControls'; +import { settingsStore } from '../../stores/settingsStore'; import { ScreenShareUserError, screenShareFailureMessage } from '../../utils/screenShareFailure'; import { t } from '../../i18n'; import { showAlert } from '../Dialog'; @@ -27,6 +28,14 @@ const SCREEN_AUDIENCE_TEXTS: AudienceTexts = { members: 'screenShare.privateUsers', more: 'screenShare.privateMore', }; +/** Each format a browser shares in, said in its own words: who can watch it is part of the choice. */ +const FORMAT_CHOICES: Readonly ChoiceCard>> = { + compatible: () => ({ value: 'compatible', icon: 'devices', title: t('screenShare.browserFormatCompatible'), + description: t('screenShare.browserFormatCompatibleDescription') }), + main: () => ({ value: 'main', icon: 'high_quality', title: t('screenShare.browserFormatMain'), + description: t('screenShare.browserFormatMainDescription') }), +}; + function liveScreens(): LiveScreens { return { shareIds: voiceStore.screenShareIds, @@ -48,6 +57,8 @@ export class BrowserScreenSharePicker { /** What the browser captures is known: the sound row shows, working or locked. */ private capabilitiesKnown = false; private available = false; + /** The formats this browser can share in, the compatible one first. */ + private formats: readonly BrowserScreenFormat[] = []; private starting = false; private createAudience(initial: ResourceAudience): ResourceAudiencePicker { @@ -85,6 +96,10 @@ export class BrowserScreenSharePicker { +