Add background segmentation mask

index.html

@@ -1688,5 +1688,104 @@ <h2>MediaStream in workers</h2>
 };</pre>
     </div>
   </section>
+  <section>
+    <h2>Background segmentation mask</h2>
+    <p>Some platforms or User Agents may provide built-in support for background segmentation of video frames, in particular for camera video streams.
+       Web applications may want to control whether background segmentation is computed at the source level and to get access to the computed segmentation masks.
+       This allows the web application for instance
+       to do custom framing or background blurring or replacement
+       while leveraging on platform computed background segmentation.
+       This allows the web application
+       to access the original unmodified frame and
+       to fine tune frame modifications based on its likings.
+       For that reason, we extend {{MediaStreamTrack}} with the following properties and {{VideoFrame}} with the following attributes.
+    </p>
+    <pre class="idl">
+partial dictionary MediaTrackSupportedConstraints {
+  boolean backgroundSegmentationMask = true;
+};
+
+partial dictionary MediaTrackConstraintSet {
+  ConstrainBoolean backgroundSegmentationMask;
+};
+
+partial dictionary MediaTrackSettings {
+  boolean backgroundSegmentationMask;
+};
+
+partial dictionary MediaTrackCapabilities {
+  sequence&lt;boolean&gt; backgroundSegmentationMask;
+};</pre>
+    <section>
+      <h3>VideoFrameMetadata dictionary extensions</h3>
+      <pre class="idl">
+partial dictionary VideoFrameMetadata {
+  ImageBitmap backgroundSegmentationMask;
+};</pre>
+      <section>
+        <h4>Dictionary {{VideoFrameMetadata}} Members</h4>
+        <dl data-link-for="VideoFrameMetadata" data-dfn-for="VideoFrameMetadata" class="dictionary-members">
+          <dt><dfn><code>backgroundSegmentationMask</code></dfn>
+              of type
+              <span class="idlMemberType">{{VideoFrameMetadata}}</span></dt>
+          <dd>
+            <p>A background segmentation mask with
+               white denoting certainly foreground,
+               black denoting certainly background and
+               grey denoting uncertainty or ambiguity with
+               light shades of grey denoting likely foreground and
+               dark shades of grey denoting likely background.</p>
+          </dd>
+        </dl>
+      </section>
+    </section>
+    <section>
+      <h3>Example</h3>
+      <pre class="example">
+// Open camera.
+const stream = await navigator.mediaDevices.getUserMedia({video: true});
+const [videoTrack] = stream.getVideoTracks();
+
+// Try to enable background segmentation mask.
+const videoCapabilities = videoTrack.getCapabilities();
+if ((videoCapabilities.backgroundSegmentationMask || []).includes(true)) {
+  await videoTrack.applyConstraints({backgroundSegmentationMask: {exact: true}});
+} else {
+  // Background segmentation mask is not supported by the platform or
+  // by the camera. Consider falling back to some other method.
+}
+
+const canvasContext = document.querySelector('canvas').getContext('2d');
+const videoProcessor = new MediaStreamTrackProcessor({track: videoTrack});
+const videoProcessorReader = videoProcessor.readable.getReader();
+
+for (;;) {
+  const {done, value: videoFrame} = await videoProcessorReader.read();
+  if (done)
+    break;
+  const {backgroundSegmentationMask} = videoFrame.metadata();
+  if (backgroundSegmentationMask) {
+    // Draw the video frame.
+    canvasContext.globalCompositeOperation = 'copy';
+    context.drawImage(videoFrame, 0, 0);
+    // Draw (or multiply with) the mask.
+    // The result is the foreground on black.
+    context.globalCompositeOperation = 'multiply';
+    canvasContext.drawImage(backgroundSegmentationMask, 0, 0);
+  }
+  else {
+    // Everything is background. Fill with black.
+    canvasContext.globalCompositeOperation = 'copy';
+    canvasContext.fillStyle = 'black';
+    canvasContext.fillRect(
+      0,
+      0,
+      canvasContext.canvas.width,
+      canvasContext.canvas.height);
+  }
+}
+      </pre>
+    </section>
+  </section>
 </body>
 </html>