fix(call): improve virtual background performance

src/components/MediaSettings/MediaSettings.vue

@@ -658,6 +658,8 @@ export default {
 					this.clearVirtualBackground()
 				}
 			} else {
+				// Disable virtual background when closing
+				this.clearVirtualBackground()
 				this.unsubscribeFromDevices(PARTICIPANT.PERMISSIONS.MAX_DEFAULT)
 			}
 		},

src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js

@@ -17,6 +17,12 @@ import WebGLCompositor from './WebGLCompositor.js'
 // Cache MediaPipe resources to avoid loading them multiple times.
 let _WasmFileset = null
 
+/**
+ * Throttle coefficient for _runInference method.
+ * Every Nth frame will be inferenced (mask calculation)
+ */
+const INFERENCE_THROTTLE_RATE = 1
+
 /**
  * Represents a modified MediaStream that applies virtual background effects
  * (blur, image, video, or video stream) using MediaPipe segmentation.
@@ -63,6 +69,8 @@ export default class VideoStreamBackgroundEffect {
 
 		this._segmentationPixelCount = this._options.width * this._options.height
 
+		this._inferenceRunning = false
+
 		this._initMediaPipe().catch((e) => console.error(e))
 
 		// Bind event handler so it is only bound once for every instance.
@@ -147,14 +155,37 @@ export default class VideoStreamBackgroundEffect {
 		}
 	}
 
+	/**
+	 * Start inference if not already running.
+	 * Deferred to next microtask to not block rendering.
+	 *
+	 * @private
+	 * @return {void}
+	 */
+	_queueInference() {
+		if (this._inferenceRunning) {
+			return
+		}
+
+		this._inferenceRunning = true
+		Promise.resolve()
+			.then(() => this._runInference())
+			.catch((error) => {
+				console.error('MediaPipe inference failed:', error)
+				this._inferenceRunning = false
+			})
+	}
+
 	/**
 	 * Run segmentation inference on the current video frame.
+	 * Update cached mask when done.
 	 *
 	 * @private
 	 * @return {Promise<void>}
 	 */
 	async _runInference() {
-		if (!this._imageSegmenter || !this._loaded) {
+		if (!this._running || !this._imageSegmenter || !this._loaded) {
+			this._inferenceRunning = false
 			return
 		}
 
@@ -165,22 +196,26 @@ export default class VideoStreamBackgroundEffect {
 				performance.now(),
 			)
 
-			if (segmentationResult.confidenceMasks && segmentationResult.confidenceMasks.length > 0) {
+			// Effect might have been stopped while inferencing
+			if (this._running && segmentationResult.confidenceMasks && segmentationResult.confidenceMasks.length > 0) {
 				this._processSegmentationResult(segmentationResult)
 			}
-
-			this.runPostProcessing()
-			this._lastFrameId = this._frameId
-		} catch (error) {
-			console.error('MediaPipe inference failed:', error)
 		} finally {
 			if (segmentationResult?.categoryMask) {
 				segmentationResult.categoryMask.close()
 			}
 
 			if (segmentationResult?.confidenceMasks?.length) {
-				segmentationResult.confidenceMasks.forEach((mask) => mask.close())
+				segmentationResult.confidenceMasks.forEach((mask) => {
+					// The mask cached in _lastMask is kept for the next postProcessing
+					if (mask === this._lastMask) {
+						return
+					}
+					mask.close()
+				})
 			}
+
+			this._inferenceRunning = false
 		}
 	}
 
@@ -258,6 +293,9 @@ export default class VideoStreamBackgroundEffect {
 			// Update segmentation mask canvas
 			this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0)
 		} else {
+			if (this._lastMask) {
+				this._lastMask.close()
+			}
 			this._lastMask = maskData
 		}
 	}
@@ -269,19 +307,16 @@ export default class VideoStreamBackgroundEffect {
 	 * @return {void}
 	 */
 	_renderMask() {
-		if (this._frameId < this._lastFrameId) {
-			console.debug('Fixing frame id, this should not happen', this._frameId, this._lastFrameId)
-			this._frameId = this._lastFrameId
-		}
-
 		// Run inference if ready
-		if (this._loaded && this._frameId === this._lastFrameId) {
+		if (this._loaded) {
 			this._frameId++
-			this._runInference().catch((e) => console.error(e))
-		} else if (this._useWebGL) {
-			this.runPostProcessing()
+			if (this._frameId % INFERENCE_THROTTLE_RATE === 0) {
+				this._queueInference()
+			}
 		}
 
+		this.runPostProcessing()
+
 		// Schedule next frame
 		this._maskFrameTimerWorker.postMessage({
 			id: SET_TIMEOUT,
@@ -422,8 +457,12 @@ export default class VideoStreamBackgroundEffect {
 			return
 		}
 
-		this._outputCanvasElement.width = width
-		this._outputCanvasElement.height = height
+		// Assigning canvas dimensions clears the canvas and reallocates its
+		// buffer even if values are unchanged, so only set them on change.
+		if (this._outputCanvasElement.width !== width || this._outputCanvasElement.height !== height) {
+			this._outputCanvasElement.width = width
+			this._outputCanvasElement.height = height
+		}
 
 		if (this._useWebGL) {
 			if (!this._glFx) {
@@ -643,7 +682,6 @@ export default class VideoStreamBackgroundEffect {
 		}
 
 		this._frameId = -1
-		this._lastFrameId = -1
 
 		this._bgChanged = true
 
@@ -669,7 +707,6 @@ export default class VideoStreamBackgroundEffect {
 		})
 
 		this._frameId = -1
-		this._lastFrameId = -1
 	}
 
 	/**
@@ -697,6 +734,11 @@ export default class VideoStreamBackgroundEffect {
 			this._glFx = null
 		}
 
+		if (this._lastMask) {
+			this._lastMask.close()
+			this._lastMask = null
+		}
+
 		this._segmentationMask = null
 		this._segmentationMaskCanvas = null
 		this._segmentationMaskCtx = null