Bug 1774300 - Add a blank VideoFrame interface r=padenot,jgilbert,emilio

This patch creates a blank class for the VideoFrame interface. The files
are generated by running `./mach build-backend && ./mach webidl-example
VideoFrame` with necessary changes to make it buildable.

The VideoFrame interface is the essential interface for W3C WebCodecs
API, used to represent the decoded video data, decoded image, and the
data ready to be encoded.

The implementations are plain blank now. They will be filled out in the
following patches.

Depends on D144771

Differential Revision: https://phabricator.services.mozilla.com/D144772

dom/media/encoder/VP8TrackEncoder.cpp

@@ -642,7 +642,7 @@ nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk& aChunk) {
   RefPtr<Image> img;
   if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) {
     if (!mMuteFrame || mMuteFrame->GetSize() != intrinsicSize) {
-      mMuteFrame = VideoFrame::CreateBlackImage(intrinsicSize);
+      mMuteFrame = mozilla::VideoFrame::CreateBlackImage(intrinsicSize);
     }
     if (!mMuteFrame) {
       VP8LOG(LogLevel::Warning, "Failed to allocate black image of size %dx%d",

dom/media/webcodecs/VideoFrame.cpp

@@ -0,0 +1,159 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/VideoFrame.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+
+namespace mozilla::dom {
+
+// Only needed for refcounted objects.
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_0(VideoFrame)
+NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoFrame)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(VideoFrame)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoFrame)
+  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+  NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+VideoFrame::VideoFrame() {
+  // Add |MOZ_COUNT_CTOR(VideoFrame);| for a non-refcounted object.
+}
+
+VideoFrame::~VideoFrame() {
+  // Add |MOZ_COUNT_DTOR(VideoFrame);| for a non-refcounted object.
+}
+
+// Add to make it buildable.
+nsIGlobalObject* VideoFrame::GetParentObject() const { return nullptr; }
+
+JSObject* VideoFrame::WrapObject(JSContext* aCx,
+                                 JS::Handle<JSObject*> aGivenProto) {
+  return VideoFrame_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// The followings are added to make it buildable.
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, HTMLImageElement& imageElement,
+    const VideoFrameInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, SVGImageElement& svgImageElement,
+    const VideoFrameInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, HTMLCanvasElement& canvasElement,
+    const VideoFrameInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, HTMLVideoElement& videoElement,
+    const VideoFrameInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, OffscreenCanvas& offscreenCanvas,
+    const VideoFrameInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(const GlobalObject& global,
+                                                     ImageBitmap& imageBitmap,
+                                                     const VideoFrameInit& init,
+                                                     ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(const GlobalObject& global,
+                                                     VideoFrame& videoFrame,
+                                                     const VideoFrameInit& init,
+                                                     ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, const ArrayBufferView& bufferView,
+    const VideoFrameBufferInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+    const GlobalObject& global, const ArrayBuffer& buffer,
+    const VideoFrameBufferInit& init, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+Nullable<VideoPixelFormat> VideoFrame::GetFormat() const { return nullptr; }
+
+uint32_t VideoFrame::CodedWidth() const { return 0; }
+
+uint32_t VideoFrame::CodedHeight() const { return 0; }
+
+already_AddRefed<DOMRectReadOnly> VideoFrame::GetCodedRect() const {
+  return nullptr;
+}
+
+already_AddRefed<DOMRectReadOnly> VideoFrame::GetVisibleRect() const {
+  return nullptr;
+}
+
+uint32_t VideoFrame::DisplayWidth() const { return 0; }
+
+uint32_t VideoFrame::DisplayHeight() const { return 0; }
+
+Nullable<uint64_t> VideoFrame::GetDuration() const { return nullptr; }
+
+Nullable<int64_t> VideoFrame::GetTimestamp() const { return nullptr; }
+
+already_AddRefed<VideoColorSpace> VideoFrame::ColorSpace() const {
+  return nullptr;
+}
+
+uint32_t VideoFrame::AllocationSize(const VideoFrameCopyToOptions& options,
+                                    ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return 0;
+}
+
+already_AddRefed<Promise> VideoFrame::CopyTo(
+    const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& destination,
+    const VideoFrameCopyToOptions& options, ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+already_AddRefed<VideoFrame> VideoFrame::Clone(ErrorResult& aRv) {
+  aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+  return nullptr;
+}
+
+void VideoFrame::Close() {}
+
+}  // namespace mozilla::dom

dom/media/webcodecs/VideoFrame.h

@@ -0,0 +1,145 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoFrame_h
+#define mozilla_dom_VideoFrame_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/TypedArray.h"  // Add to make it buildable.
+#include "nsCycleCollectionParticipant.h"
+#include "nsWrapperCache.h"
+
+class nsIGlobalObject;  // Add to make it buildable.
+
+namespace mozilla {
+namespace dom {
+
+class DOMRectReadOnly;
+class HTMLCanvasElement;  // Add to make it buildable.
+class HTMLImageElement;   // Add to make it buildable.
+class HTMLVideoElement;   // Add to make it buildable.
+class ImageBitmap;        // Add to make it buildable.
+class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class OffscreenCanvas;  // Add to make it buildable.
+class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class Promise;
+class SVGImageElement;  // Add to make it buildable.
+class VideoColorSpace;
+class VideoFrame;
+enum class VideoPixelFormat : uint8_t;  // Add to make it buildable.
+struct VideoFrameBufferInit;            // Add to make it buildable.
+struct VideoFrameInit;                  // Add to make it buildable.
+struct VideoFrameCopyToOptions;
+
+}  // namespace dom
+}  // namespace mozilla
+
+namespace mozilla::dom {
+
+class VideoFrame final
+    : public nsISupports /* or NonRefcountedDOMObject if this is a
+                            non-refcounted object */
+    ,
+      public nsWrapperCache /* Change wrapperCache in the binding configuration
+                               if you don't want this */
+{
+ public:
+  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+  NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame)
+
+ public:
+  VideoFrame();
+
+ protected:
+  ~VideoFrame();
+
+ public:
+  // This should return something that eventually allows finding a
+  // path to the global this object is associated with.  Most simply,
+  // returning an actual global works.
+  nsIGlobalObject* GetParentObject() const;
+
+  JSObject* WrapObject(JSContext* aCx,
+                       JS::Handle<JSObject*> aGivenProto) override;
+
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, HTMLImageElement& imageElement,
+      const VideoFrameInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, SVGImageElement& svgImageElement,
+      const VideoFrameInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, HTMLCanvasElement& canvasElement,
+      const VideoFrameInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, HTMLVideoElement& videoElement,
+      const VideoFrameInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, OffscreenCanvas& offscreenCanvas,
+      const VideoFrameInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(const GlobalObject& global,
+                                                  ImageBitmap& imageBitmap,
+                                                  const VideoFrameInit& init,
+                                                  ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(const GlobalObject& global,
+                                                  VideoFrame& videoFrame,
+                                                  const VideoFrameInit& init,
+                                                  ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, const ArrayBufferView& bufferView,
+      const VideoFrameBufferInit& init, ErrorResult& aRv);
+  static already_AddRefed<VideoFrame> Constructor(
+      const GlobalObject& global, const ArrayBuffer& buffer,
+      const VideoFrameBufferInit& init, ErrorResult& aRv);
+
+  Nullable<VideoPixelFormat> GetFormat() const;
+
+  uint32_t CodedWidth() const;
+
+  uint32_t CodedHeight() const;
+
+  // Return a raw pointer here to avoid refcounting, but make sure it's safe
+  // (the object should be kept alive by the callee).
+  already_AddRefed<DOMRectReadOnly> GetCodedRect() const;
+
+  // Return a raw pointer here to avoid refcounting, but make sure it's safe
+  // (the object should be kept alive by the callee).
+  already_AddRefed<DOMRectReadOnly> GetVisibleRect() const;
+
+  uint32_t DisplayWidth() const;
+
+  uint32_t DisplayHeight() const;
+
+  Nullable<uint64_t> GetDuration() const;
+
+  Nullable<int64_t> GetTimestamp() const;
+
+  // Return a raw pointer here to avoid refcounting, but make sure it's safe
+  // (the object should be kept alive by the callee).
+  already_AddRefed<VideoColorSpace> ColorSpace() const;
+
+  uint32_t AllocationSize(const VideoFrameCopyToOptions& options,
+                          ErrorResult& aRv);
+
+  // Return a raw pointer here to avoid refcounting, but make sure it's safe
+  // (the object should be kept alive by the callee).
+  already_AddRefed<Promise> CopyTo(
+      const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& destination,
+      const VideoFrameCopyToOptions& options, ErrorResult& aRv);
+
+  // Return a raw pointer here to avoid refcounting, but make sure it's safe
+  // (the object should be kept alive by the callee).
+  already_AddRefed<VideoFrame> Clone(ErrorResult& aRv);
+
+  void Close();
+};
+
+}  // namespace mozilla::dom
+
+#endif  // mozilla_dom_VideoFrame_h

dom/media/webcodecs/moz.build

@@ -6,10 +6,12 @@
 
 EXPORTS.mozilla.dom += [
     "VideoColorSpace.h",
+    "VideoFrame.h",
 ]
 
 UNIFIED_SOURCES += [
     "VideoColorSpace.cpp",
+    "VideoFrame.cpp",
 ]
 
 FINAL_LIBRARY = "xul"

dom/webidl/VideoFrame.webidl

@@ -0,0 +1,136 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://w3c.github.io/webcodecs/#videoframe
+ */
+
+enum AlphaOption {
+  "keep",
+  "discard",
+};
+
+[Exposed=(Window,DedicatedWorker) /* , Serializable (bug 1774302), Transferable (bug 1774306) */, Pref="dom.media.webcodecs.enabled"]
+interface VideoFrame {
+  // The constructors should be shorten to:
+  //   ```
+  //   constructor([AllowShared] BufferSource data, VideoFrameBufferInit init);
+  //   constructor(CanvasImageSource image, optional VideoFrameInit init = {});
+  //   ```
+  // However, `[AllowShared] BufferSource` doesn't work for now (bug 1696216), and
+  // `No support for unions as distinguishing arguments yet` error occurs when using
+  //   `constructor(CanvasImageSource image, optional VideoFrameInit init = {})` and
+  //   `constructor(([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) data, VideoFrameBufferInit init)`
+  // at the same time (bug 1786410).
+  [Throws]
+  constructor(HTMLImageElement imageElement, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(SVGImageElement svgImageElement, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(HTMLCanvasElement canvasElement, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(HTMLVideoElement videoElement, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(OffscreenCanvas offscreenCanvas, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(ImageBitmap imageBitmap, optional VideoFrameInit init = {});
+  [Throws]
+  constructor(VideoFrame videoFrame, optional VideoFrameInit init = {});
+  [Throws]
+  constructor([AllowShared] ArrayBufferView bufferView, VideoFrameBufferInit init);
+  [Throws]
+  constructor([AllowShared] ArrayBuffer buffer, VideoFrameBufferInit init);
+
+
+  readonly attribute VideoPixelFormat? format;
+  readonly attribute unsigned long codedWidth;
+  readonly attribute unsigned long codedHeight;
+  readonly attribute DOMRectReadOnly? codedRect;
+  readonly attribute DOMRectReadOnly? visibleRect;
+  readonly attribute unsigned long displayWidth;
+  readonly attribute unsigned long displayHeight;
+  readonly attribute unsigned long long? duration;  // microseconds
+  readonly attribute long long? timestamp;          // microseconds
+  readonly attribute VideoColorSpace colorSpace;
+
+  [Throws]
+  unsigned long allocationSize(
+      optional VideoFrameCopyToOptions options = {});
+  [Throws]
+  Promise<sequence<PlaneLayout>> copyTo(
+      // bug 1696216: Should be `copyTo([AllowShared] BufferSource destination, ...)`
+      ([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) destination,
+      optional VideoFrameCopyToOptions options = {});
+  [Throws]
+  VideoFrame clone();
+  undefined close();
+};
+
+dictionary VideoFrameInit {
+  unsigned long long duration;  // microseconds
+  long long timestamp;          // microseconds
+  AlphaOption alpha = "keep";
+
+  // Default matches image. May be used to efficiently crop. Will trigger
+  // new computation of displayWidth and displayHeight using image’s pixel
+  // aspect ratio unless an explicit displayWidth and displayHeight are given.
+  DOMRectInit visibleRect;
+
+  // Default matches image unless visibleRect is provided.
+  [EnforceRange] unsigned long displayWidth;
+  [EnforceRange] unsigned long displayHeight;
+};
+
+dictionary VideoFrameBufferInit {
+  required VideoPixelFormat format;
+  required [EnforceRange] unsigned long codedWidth;
+  required [EnforceRange] unsigned long codedHeight;
+  required [EnforceRange] long long timestamp;  // microseconds
+  [EnforceRange] unsigned long long duration;   // microseconds
+
+  // Default layout is tightly-packed.
+  sequence<PlaneLayout> layout;
+
+  // Default visible rect is coded size positioned at (0,0)
+  DOMRectInit visibleRect;
+
+  // Default display dimensions match visibleRect.
+  [EnforceRange] unsigned long displayWidth;
+  [EnforceRange] unsigned long displayHeight;
+
+  VideoColorSpaceInit colorSpace;
+};
+
+dictionary VideoFrameCopyToOptions {
+  DOMRectInit rect;
+  sequence<PlaneLayout> layout;
+};
+
+dictionary PlaneLayout {
+  // TODO: https://github.com/w3c/webcodecs/pull/488
+  required [EnforceRange] unsigned long offset;
+  required [EnforceRange] unsigned long stride;
+};
+
+enum VideoPixelFormat {
+  // 4:2:0 Y, U, V
+  "I420",
+  // 4:2:0 Y, U, V, A
+  "I420A",
+  // 4:2:2 Y, U, V
+  "I422",
+  // 4:4:4 Y, U, V
+  "I444",
+  // 4:2:0 Y, UV
+  "NV12",
+  // 32bpp RGBA
+  "RGBA",
+  // 32bpp RGBX (opaque)
+  "RGBX",
+  // 32bpp BGRA
+  "BGRA",
+  // 32bpp BGRX (opaque)
+  "BGRX",
+};

dom/webidl/moz.build

@@ -976,6 +976,7 @@ WEBIDL_FILES = [
     "URLSearchParams.webidl",
     "ValidityState.webidl",
     "VideoColorSpace.webidl",
+    "VideoFrame.webidl",
     "VideoPlaybackQuality.webidl",
     "VideoTrack.webidl",
     "VideoTrackList.webidl",

testing/web-platform/meta/webcodecs/videoFrame-construction.any.js.ini

@@ -1,4 +1,6 @@
 [videoFrame-construction.any.worker.html]
+  prefs: [dom.media.webcodecs.enabled:true]
+
   [Test we can construct a VideoFrame.]
     expected: FAIL
 
@@ -88,6 +90,8 @@
 
 
 [videoFrame-construction.any.html]
+  prefs: [dom.media.webcodecs.enabled:true]
+
   [Test we can construct a VideoFrame.]
     expected: FAIL
 

testing/web-platform/meta/webcodecs/videoFrame-construction.crossOriginIsolated.https.any.js.ini

@@ -1,4 +1,5 @@
 [videoFrame-construction.crossOriginIsolated.https.any.worker.html]
+  prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
   [Test SharedArrayBuffer constructed I420 VideoFrame]
@@ -9,6 +10,7 @@
 
 
 [videoFrame-construction.crossOriginIsolated.https.any.html]
+  prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and debug and not fission and not swgl: [OK, TIMEOUT]
   [Test SharedArrayBuffer constructed I420 VideoFrame]

testing/web-platform/meta/webcodecs/videoFrame-construction.window.js.ini

@@ -1,4 +1,5 @@
 [videoFrame-construction.window.html]
+  prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
   [Test that timestamp is required when constructing VideoFrame from HTMLImageElement]

testing/web-platform/meta/webcodecs/videoFrame-copyTo.any.js.ini

@@ -1,4 +1,6 @@
 [videoFrame-copyTo.any.html]
+  prefs: [dom.media.webcodecs.enabled:true]
+
   [Test RGBA frame.]
     expected: FAIL
 
@@ -46,6 +48,8 @@
 
 
 [videoFrame-copyTo.any.worker.html]
+  prefs: [dom.media.webcodecs.enabled:true]
+
   [Test RGBA frame.]
     expected: FAIL
 

testing/web-platform/meta/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.js.ini

@@ -1,4 +1,5 @@
 [videoFrame-copyTo.crossOriginIsolated.https.any.html]
+  prefs: [dom.media.webcodecs.enabled:true]
   [Test copying I420 frame to SharedArrayBuffer.]
     expected: FAIL
 
@@ -7,6 +8,7 @@
 
 
 [videoFrame-copyTo.crossOriginIsolated.https.any.worker.html]
+  prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
   [Test copying I420 frame to SharedArrayBuffer.]