From 102ef57e7422410aa2af790c8dc27f422ff87d36 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Wed, 3 Jun 2026 11:02:05 +0200 Subject: [PATCH 1/3] fix(virtual-bg): stop effect when device checker is not in use Signed-off-by: Maksim Sukharev --- src/components/MediaSettings/MediaSettings.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/components/MediaSettings/MediaSettings.vue b/src/components/MediaSettings/MediaSettings.vue index ebba99f0a84..5694a116392 100644 --- a/src/components/MediaSettings/MediaSettings.vue +++ b/src/components/MediaSettings/MediaSettings.vue @@ -658,6 +658,8 @@ export default { this.clearVirtualBackground() } } else { + // Disable virtual background when closing + this.clearVirtualBackground() this.unsubscribeFromDevices(PARTICIPANT.PERMISSIONS.MAX_DEFAULT) } }, From 009dc8c3f0c3936afa7edb67d6edb8aad9491046 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Wed, 10 Jun 2026 16:45:50 +0200 Subject: [PATCH 2/3] fix(virtual-bg): defer mask inferencing to not block rendering - mask inference is queued as a microtask - post process frames on every tick from the last cached mask, without waiting for the inference including the 2D canvas fallback path - keep the cached mask open after inference - remove _lasFrameId tracking (redundant) Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Maksim Sukharev --- .../VideoStreamBackgroundEffect.js | 76 ++++++++++++++----- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js b/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js index f0176d04505..dbb1e38c30d 100644 --- a/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js +++ b/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js @@ -17,6 +17,12 @@ import WebGLCompositor from './WebGLCompositor.js' // Cache MediaPipe resources to avoid loading them multiple times. let _WasmFileset = null +/** + * Throttle coefficient for _runInference method. + * Every Nth frame will be inferenced (mask calculation) + */ +const INFERENCE_THROTTLE_RATE = 1 + /** * Represents a modified MediaStream that applies virtual background effects * (blur, image, video, or video stream) using MediaPipe segmentation. @@ -63,6 +69,8 @@ export default class VideoStreamBackgroundEffect { this._segmentationPixelCount = this._options.width * this._options.height + this._inferenceRunning = false + this._initMediaPipe().catch((e) => console.error(e)) // Bind event handler so it is only bound once for every instance. @@ -147,14 +155,37 @@ export default class VideoStreamBackgroundEffect { } } + /** + * Start inference if not already running. + * Deferred to next microtask to not block rendering. + * + * @private + * @return {void} + */ + _queueInference() { + if (this._inferenceRunning) { + return + } + + this._inferenceRunning = true + Promise.resolve() + .then(() => this._runInference()) + .catch((error) => { + console.error('MediaPipe inference failed:', error) + this._inferenceRunning = false + }) + } + /** * Run segmentation inference on the current video frame. + * Update cached mask when done. * * @private * @return {Promise} */ async _runInference() { - if (!this._imageSegmenter || !this._loaded) { + if (!this._running || !this._imageSegmenter || !this._loaded) { + this._inferenceRunning = false return } @@ -165,22 +196,26 @@ export default class VideoStreamBackgroundEffect { performance.now(), ) - if (segmentationResult.confidenceMasks && segmentationResult.confidenceMasks.length > 0) { + // Effect might have been stopped while inferencing + if (this._running && segmentationResult.confidenceMasks && segmentationResult.confidenceMasks.length > 0) { this._processSegmentationResult(segmentationResult) } - - this.runPostProcessing() - this._lastFrameId = this._frameId - } catch (error) { - console.error('MediaPipe inference failed:', error) } finally { if (segmentationResult?.categoryMask) { segmentationResult.categoryMask.close() } if (segmentationResult?.confidenceMasks?.length) { - segmentationResult.confidenceMasks.forEach((mask) => mask.close()) + segmentationResult.confidenceMasks.forEach((mask) => { + // The mask cached in _lastMask is kept for the next postProcessing + if (mask === this._lastMask) { + return + } + mask.close() + }) } + + this._inferenceRunning = false } } @@ -258,6 +293,9 @@ export default class VideoStreamBackgroundEffect { // Update segmentation mask canvas this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0) } else { + if (this._lastMask) { + this._lastMask.close() + } this._lastMask = maskData } } @@ -269,19 +307,16 @@ export default class VideoStreamBackgroundEffect { * @return {void} */ _renderMask() { - if (this._frameId < this._lastFrameId) { - console.debug('Fixing frame id, this should not happen', this._frameId, this._lastFrameId) - this._frameId = this._lastFrameId - } - // Run inference if ready - if (this._loaded && this._frameId === this._lastFrameId) { + if (this._loaded) { this._frameId++ - this._runInference().catch((e) => console.error(e)) - } else if (this._useWebGL) { - this.runPostProcessing() + if (this._frameId % INFERENCE_THROTTLE_RATE === 0) { + this._queueInference() + } } + this.runPostProcessing() + // Schedule next frame this._maskFrameTimerWorker.postMessage({ id: SET_TIMEOUT, @@ -643,7 +678,6 @@ export default class VideoStreamBackgroundEffect { } this._frameId = -1 - this._lastFrameId = -1 this._bgChanged = true @@ -669,7 +703,6 @@ export default class VideoStreamBackgroundEffect { }) this._frameId = -1 - this._lastFrameId = -1 } /** @@ -697,6 +730,11 @@ export default class VideoStreamBackgroundEffect { this._glFx = null } + if (this._lastMask) { + this._lastMask.close() + this._lastMask = null + } + this._segmentationMask = null this._segmentationMaskCanvas = null this._segmentationMaskCtx = null From 0362ce6e8375a87fdea2fbab216f3d2eb04dfb85 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Wed, 10 Jun 2026 16:47:40 +0200 Subject: [PATCH 3/3] perf(virtual-bg): only resize output canvas when dimensions change - Assigning width/height clears the canvas and reallocates the buffer even if values are unchanged Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Maksim Sukharev --- .../virtual-background/VideoStreamBackgroundEffect.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js b/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js index dbb1e38c30d..ea039cd6bf6 100644 --- a/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js +++ b/src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js @@ -457,8 +457,12 @@ export default class VideoStreamBackgroundEffect { return } - this._outputCanvasElement.width = width - this._outputCanvasElement.height = height + // Assigning canvas dimensions clears the canvas and reallocates its + // buffer even if values are unchanged, so only set them on change. + if (this._outputCanvasElement.width !== width || this._outputCanvasElement.height !== height) { + this._outputCanvasElement.width = width + this._outputCanvasElement.height = height + } if (this._useWebGL) { if (!this._glFx) {