Skip to content

fix(call): improve virtual background performance #17450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Antreesy wants to merge 3 commits into
base: main
Choose a base branch
from
+120 −18
Open

fix(call): improve virtual background performance #17450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
180e2af
fix(virtual-bg): use requestVideoFrameCallback when available
Antreesy Sep 1, 2026
0c10766
fix(virtual-bg): keep _lastMask open while being reused
Antreesy Sep 1, 2026
941c071
fix(virtual-bg): avoid GPU/canvas reallocation every frame
Antreesy Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 91 additions & 10 deletions src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ export default class VideoStreamBackgroundEffect {
// _renderMask: Function;
// _virtualImage: HTMLImageElement;
// _virtualVideo: HTMLVideoElement;
// _useRVFC: boolean;
// _rvfcHandle: number;
// _lastInferenceTimestamp: number;

/**
* Create a new background effect processor.
Expand Down Expand Up @@ -81,6 +84,9 @@ export default class VideoStreamBackgroundEffect {
this._inputVideoElement = document.createElement('video')
this._bgChanged = false
this._prevBgMode = null
this._useRVFC = undefined
this._rvfcHandle = undefined
this._lastInferenceTimestamp = -1
}

/**
Expand Down Expand Up @@ -130,6 +136,9 @@ export default class VideoStreamBackgroundEffect {
'./vendor/models/selfie_segmenter.tflite',
import.meta.url,
).pathname,
// delegate: 'CPU' was tried as a GPU-load reduction; it
// made things worse (no GPU win, added main-thread
// stalls). Keeping GPU.
delegate: 'GPU',
},
runningMode: 'VIDEO',
Expand Down Expand Up @@ -157,11 +166,15 @@ export default class VideoStreamBackgroundEffect {
return
}

// MediaPipe VIDEO mode requires strictly increasing timestamps.
const now = performance.now()
this._lastInferenceTimestamp = now > this._lastInferenceTimestamp ? now : this._lastInferenceTimestamp + 1

let segmentationResult
try {
segmentationResult = await this._imageSegmenter.segmentForVideo(
this._inputVideoElement,
performance.now(),
this._lastInferenceTimestamp,
)

if (segmentationResult.confidenceMasks && segmentationResult.confidenceMasks.length > 0) {
Expand All @@ -178,7 +191,13 @@ export default class VideoStreamBackgroundEffect {
}

if (segmentationResult?.confidenceMasks?.length) {
segmentationResult.confidenceMasks.forEach((mask) => mask.close())
// The mask kept as _lastMask (WebGL path) must stay open for
// reuse on stale redraws; see _processSegmentationResult().
segmentationResult.confidenceMasks.forEach((mask) => {
if (mask !== this._lastMask) {
mask.close()
}
})
}
}
}
Expand Down Expand Up @@ -257,6 +276,11 @@ export default class VideoStreamBackgroundEffect {
// Update segmentation mask canvas
this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0)
} else {
// Close the previous mask only once replaced; it stays open
// until then for reuse on stale redraws (see _runInference()).
if (this._lastMask && this._lastMask !== maskData) {
this._lastMask.close()
}
this._lastMask = maskData
}
}
Expand All @@ -281,14 +305,58 @@ export default class VideoStreamBackgroundEffect {
this.runPostProcessing()
}

// Schedule next frame
// rVFC reschedules itself in _startFrameLoop(); only the fallback needs this.
if (!this._useRVFC) {
this._scheduleFallbackTick()
}
}

/**
* Schedules the next _renderMask() tick via the fixed-interval Worker
* timer (fallback for browsers without requestVideoFrameCallback).
*
* @private
* @return {void}
*/
_scheduleFallbackTick() {
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / this._frameRate,
message: 'this._maskFrameTimerWorker',
})
}

/**
* Starts the loop that triggers _renderMask() once per input video frame.
*
* Uses requestVideoFrameCallback where available (fires exactly on new
* frames, keeps working in background tabs); falls back to the
* Worker-based timer otherwise.
*
* @private
* @return {void}
*/
_startFrameLoop() {
if ('requestVideoFrameCallback' in this._inputVideoElement) {
this._useRVFC = true

const onVideoFrame = () => {
this._renderMask()

if (this._running) {
this._rvfcHandle = this._inputVideoElement.requestVideoFrameCallback(onVideoFrame)
}
}

this._rvfcHandle = this._inputVideoElement.requestVideoFrameCallback(onVideoFrame)

return
}

this._useRVFC = false
this._scheduleFallbackTick()
}

/**
* Handle timer worker ticks to schedule mask rendering.
*
Expand Down Expand Up @@ -421,8 +489,13 @@ export default class VideoStreamBackgroundEffect {
return
}

this._outputCanvasElement.width = width
this._outputCanvasElement.height = height
// Assigning canvas dimensions clears and reallocates its backing
// buffer even when the values are unchanged, so only set them on
// an actual resize.
if (this._outputCanvasElement.width !== width || this._outputCanvasElement.height !== height) {
this._outputCanvasElement.width = width
this._outputCanvasElement.height = height
}

if (this._useWebGL) {
if (!this._glFx) {
Expand Down Expand Up @@ -629,11 +702,7 @@ export default class VideoStreamBackgroundEffect {
this._inputVideoElement.autoplay = true
this._inputVideoElement.srcObject = this._stream
this._inputVideoElement.onloadeddata = () => {
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / this._frameRate,
message: 'this._maskFrameTimerWorker',
})
this._startFrameLoop()
this._inputVideoElement.onloadeddata = null
}

Expand Down Expand Up @@ -679,6 +748,18 @@ export default class VideoStreamBackgroundEffect {
stopEffect() {
this._running = false

// The "_running" check in onVideoFrame only stops future callbacks;
// one may already be pending, so cancel it explicitly.
if (this._rvfcHandle !== undefined) {
this._inputVideoElement.cancelVideoFrameCallback(this._rvfcHandle)
this._rvfcHandle = undefined
}

if (this._lastMask) {
this._lastMask.close()
this._lastMask = null
}

if (this._maskFrameTimerWorker) {
this._maskFrameTimerWorker.postMessage({
id: CLEAR_TIMEOUT,
Expand Down
37 changes: 29 additions & 8 deletions src/utils/media/effects/virtual-background/WebGLCompositor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,14 @@ export default class WebGLCompositor {
this.bgScale = [1.0, 1.0]
this.lastOutW = 0
this.lastOutH = 0

// Tracks the size texMaskFiltered/texBlurred{1,2} are currently
// allocated at, so their storage is only reallocated on resize
// instead of on every render() call.
this._maskFilteredW = 0
this._maskFilteredH = 0
this._blurW = 0
this._blurH = 0
}

/**
Expand Down Expand Up @@ -527,11 +535,17 @@ export default class WebGLCompositor {
const texelWidth = 1 / blurWidth
const texelHeight = 1 / blurHeight

// Allocate blur textures
gl.bindTexture(gl.TEXTURE_2D, this.texBlurred1)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurWidth, blurHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
gl.bindTexture(gl.TEXTURE_2D, this.texBlurred2)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurWidth, blurHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
// Allocate blur textures, only reallocating storage when the blur
// size actually changes (see the matching comment on texMaskFiltered
// in render() for why this check matters).
if (this._blurW !== blurWidth || this._blurH !== blurHeight) {
gl.bindTexture(gl.TEXTURE_2D, this.texBlurred1)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurWidth, blurHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
gl.bindTexture(gl.TEXTURE_2D, this.texBlurred2)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurWidth, blurHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
this._blurW = blurWidth
this._blurH = blurHeight
}

// Setup FBOs
gl.bindFramebuffer(gl.FRAMEBUFFER, this.fboBlur1)
Expand Down Expand Up @@ -718,9 +732,16 @@ export default class WebGLCompositor {
return
}

// Allocate mask filtered texture
gl.bindTexture(gl.TEXTURE_2D, this.texMaskFiltered)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, outW, outH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
// Allocate mask filtered texture, only reallocating storage when the
// output size actually changes (texImage2D(null) forces the GPU to
// allocate new backing storage every call, which is wasted work when
// called every frame at an unchanged size).
if (this._maskFilteredW !== outW || this._maskFilteredH !== outH) {
gl.bindTexture(gl.TEXTURE_2D, this.texMaskFiltered)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, outW, outH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
this._maskFilteredW = outW
this._maskFilteredH = outH
}

// Upload and process mask
if (mask) {
Expand Down
Loading