diff --git a/dom/media/encoder/VP8TrackEncoder.cpp b/dom/media/encoder/VP8TrackEncoder.cpp index 82b2e280060b..84a98c878006 100644 --- a/dom/media/encoder/VP8TrackEncoder.cpp +++ b/dom/media/encoder/VP8TrackEncoder.cpp @@ -642,7 +642,7 @@ nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk& aChunk) { RefPtr img; if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) { if (!mMuteFrame || mMuteFrame->GetSize() != intrinsicSize) { - mMuteFrame = VideoFrame::CreateBlackImage(intrinsicSize); + mMuteFrame = mozilla::VideoFrame::CreateBlackImage(intrinsicSize); } if (!mMuteFrame) { VP8LOG(LogLevel::Warning, "Failed to allocate black image of size %dx%d", diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp new file mode 100644 index 000000000000..a989b83db9cf --- /dev/null +++ b/dom/media/webcodecs/VideoFrame.cpp @@ -0,0 +1,159 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" + +namespace mozilla::dom { + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_0(VideoFrame) +NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoFrame) +NS_IMPL_CYCLE_COLLECTING_RELEASE(VideoFrame) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoFrame) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +VideoFrame::VideoFrame() { + // Add |MOZ_COUNT_CTOR(VideoFrame);| for a non-refcounted object. +} + +VideoFrame::~VideoFrame() { + // Add |MOZ_COUNT_DTOR(VideoFrame);| for a non-refcounted object. +} + +// Add to make it buildable. +nsIGlobalObject* VideoFrame::GetParentObject() const { return nullptr; } + +JSObject* VideoFrame::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return VideoFrame_Binding::Wrap(aCx, this, aGivenProto); +} + +// The followings are added to make it buildable. + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, HTMLImageElement& imageElement, + const VideoFrameInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, SVGImageElement& svgImageElement, + const VideoFrameInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, HTMLCanvasElement& canvasElement, + const VideoFrameInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, HTMLVideoElement& videoElement, + const VideoFrameInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, OffscreenCanvas& offscreenCanvas, + const VideoFrameInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor(const GlobalObject& global, + ImageBitmap& imageBitmap, + const VideoFrameInit& init, + ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor(const GlobalObject& global, + VideoFrame& videoFrame, + const VideoFrameInit& init, + ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, const ArrayBufferView& bufferView, + const VideoFrameBufferInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +/* static */ +already_AddRefed VideoFrame::Constructor( + const GlobalObject& global, const ArrayBuffer& buffer, + const VideoFrameBufferInit& init, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +Nullable VideoFrame::GetFormat() const { return nullptr; } + +uint32_t VideoFrame::CodedWidth() const { return 0; } + +uint32_t VideoFrame::CodedHeight() const { return 0; } + +already_AddRefed VideoFrame::GetCodedRect() const { + return nullptr; +} + +already_AddRefed VideoFrame::GetVisibleRect() const { + return nullptr; +} + +uint32_t VideoFrame::DisplayWidth() const { return 0; } + +uint32_t VideoFrame::DisplayHeight() const { return 0; } + +Nullable VideoFrame::GetDuration() const { return nullptr; } + +Nullable VideoFrame::GetTimestamp() const { return nullptr; } + +already_AddRefed VideoFrame::ColorSpace() const { + return nullptr; +} + +uint32_t VideoFrame::AllocationSize(const VideoFrameCopyToOptions& options, + ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return 0; +} + +already_AddRefed VideoFrame::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& destination, + const VideoFrameCopyToOptions& options, ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +already_AddRefed VideoFrame::Clone(ErrorResult& aRv) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; +} + +void VideoFrame::Close() {} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoFrame.h b/dom/media/webcodecs/VideoFrame.h new file mode 100644 index 000000000000..d1aabc4b3d6d --- /dev/null +++ b/dom/media/webcodecs/VideoFrame.h @@ -0,0 +1,145 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoFrame_h +#define mozilla_dom_VideoFrame_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/TypedArray.h" // Add to make it buildable. +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; // Add to make it buildable. + +namespace mozilla { +namespace dom { + +class DOMRectReadOnly; +class HTMLCanvasElement; // Add to make it buildable. +class HTMLImageElement; // Add to make it buildable. +class HTMLVideoElement; // Add to make it buildable. +class ImageBitmap; // Add to make it buildable. +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OffscreenCanvas; // Add to make it buildable. +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class Promise; +class SVGImageElement; // Add to make it buildable. +class VideoColorSpace; +class VideoFrame; +enum class VideoPixelFormat : uint8_t; // Add to make it buildable. +struct VideoFrameBufferInit; // Add to make it buildable. +struct VideoFrameInit; // Add to make it buildable. +struct VideoFrameCopyToOptions; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +class VideoFrame final + : public nsISupports /* or NonRefcountedDOMObject if this is a + non-refcounted object */ + , + public nsWrapperCache /* Change wrapperCache in the binding configuration + if you don't want this */ +{ + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame) + + public: + VideoFrame(); + + protected: + ~VideoFrame(); + + public: + // This should return something that eventually allows finding a + // path to the global this object is associated with. Most simply, + // returning an actual global works. + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& global, HTMLImageElement& imageElement, + const VideoFrameInit& init, ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, SVGImageElement& svgImageElement, + const VideoFrameInit& init, ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, HTMLCanvasElement& canvasElement, + const VideoFrameInit& init, ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, HTMLVideoElement& videoElement, + const VideoFrameInit& init, ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, OffscreenCanvas& offscreenCanvas, + const VideoFrameInit& init, ErrorResult& aRv); + static already_AddRefed Constructor(const GlobalObject& global, + ImageBitmap& imageBitmap, + const VideoFrameInit& init, + ErrorResult& aRv); + static already_AddRefed Constructor(const GlobalObject& global, + VideoFrame& videoFrame, + const VideoFrameInit& init, + ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, const ArrayBufferView& bufferView, + const VideoFrameBufferInit& init, ErrorResult& aRv); + static already_AddRefed Constructor( + const GlobalObject& global, const ArrayBuffer& buffer, + const VideoFrameBufferInit& init, ErrorResult& aRv); + + Nullable GetFormat() const; + + uint32_t CodedWidth() const; + + uint32_t CodedHeight() const; + + // Return a raw pointer here to avoid refcounting, but make sure it's safe + // (the object should be kept alive by the callee). + already_AddRefed GetCodedRect() const; + + // Return a raw pointer here to avoid refcounting, but make sure it's safe + // (the object should be kept alive by the callee). + already_AddRefed GetVisibleRect() const; + + uint32_t DisplayWidth() const; + + uint32_t DisplayHeight() const; + + Nullable GetDuration() const; + + Nullable GetTimestamp() const; + + // Return a raw pointer here to avoid refcounting, but make sure it's safe + // (the object should be kept alive by the callee). + already_AddRefed ColorSpace() const; + + uint32_t AllocationSize(const VideoFrameCopyToOptions& options, + ErrorResult& aRv); + + // Return a raw pointer here to avoid refcounting, but make sure it's safe + // (the object should be kept alive by the callee). + already_AddRefed CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& destination, + const VideoFrameCopyToOptions& options, ErrorResult& aRv); + + // Return a raw pointer here to avoid refcounting, but make sure it's safe + // (the object should be kept alive by the callee). + already_AddRefed Clone(ErrorResult& aRv); + + void Close(); +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoFrame_h diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build index 9ab38e0d8817..01d1706f3609 100644 --- a/dom/media/webcodecs/moz.build +++ b/dom/media/webcodecs/moz.build @@ -6,10 +6,12 @@ EXPORTS.mozilla.dom += [ "VideoColorSpace.h", + "VideoFrame.h", ] UNIFIED_SOURCES += [ "VideoColorSpace.cpp", + "VideoFrame.cpp", ] FINAL_LIBRARY = "xul" diff --git a/dom/webidl/VideoFrame.webidl b/dom/webidl/VideoFrame.webidl new file mode 100644 index 000000000000..9d23f165fd26 --- /dev/null +++ b/dom/webidl/VideoFrame.webidl @@ -0,0 +1,136 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://w3c.github.io/webcodecs/#videoframe + */ + +enum AlphaOption { + "keep", + "discard", +}; + +[Exposed=(Window,DedicatedWorker) /* , Serializable (bug 1774302), Transferable (bug 1774306) */, Pref="dom.media.webcodecs.enabled"] +interface VideoFrame { + // The constructors should be shorten to: + // ``` + // constructor([AllowShared] BufferSource data, VideoFrameBufferInit init); + // constructor(CanvasImageSource image, optional VideoFrameInit init = {}); + // ``` + // However, `[AllowShared] BufferSource` doesn't work for now (bug 1696216), and + // `No support for unions as distinguishing arguments yet` error occurs when using + // `constructor(CanvasImageSource image, optional VideoFrameInit init = {})` and + // `constructor(([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) data, VideoFrameBufferInit init)` + // at the same time (bug 1786410). + [Throws] + constructor(HTMLImageElement imageElement, optional VideoFrameInit init = {}); + [Throws] + constructor(SVGImageElement svgImageElement, optional VideoFrameInit init = {}); + [Throws] + constructor(HTMLCanvasElement canvasElement, optional VideoFrameInit init = {}); + [Throws] + constructor(HTMLVideoElement videoElement, optional VideoFrameInit init = {}); + [Throws] + constructor(OffscreenCanvas offscreenCanvas, optional VideoFrameInit init = {}); + [Throws] + constructor(ImageBitmap imageBitmap, optional VideoFrameInit init = {}); + [Throws] + constructor(VideoFrame videoFrame, optional VideoFrameInit init = {}); + [Throws] + constructor([AllowShared] ArrayBufferView bufferView, VideoFrameBufferInit init); + [Throws] + constructor([AllowShared] ArrayBuffer buffer, VideoFrameBufferInit init); + + + readonly attribute VideoPixelFormat? format; + readonly attribute unsigned long codedWidth; + readonly attribute unsigned long codedHeight; + readonly attribute DOMRectReadOnly? codedRect; + readonly attribute DOMRectReadOnly? visibleRect; + readonly attribute unsigned long displayWidth; + readonly attribute unsigned long displayHeight; + readonly attribute unsigned long long? duration; // microseconds + readonly attribute long long? timestamp; // microseconds + readonly attribute VideoColorSpace colorSpace; + + [Throws] + unsigned long allocationSize( + optional VideoFrameCopyToOptions options = {}); + [Throws] + Promise> copyTo( + // bug 1696216: Should be `copyTo([AllowShared] BufferSource destination, ...)` + ([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) destination, + optional VideoFrameCopyToOptions options = {}); + [Throws] + VideoFrame clone(); + undefined close(); +}; + +dictionary VideoFrameInit { + unsigned long long duration; // microseconds + long long timestamp; // microseconds + AlphaOption alpha = "keep"; + + // Default matches image. May be used to efficiently crop. Will trigger + // new computation of displayWidth and displayHeight using image’s pixel + // aspect ratio unless an explicit displayWidth and displayHeight are given. + DOMRectInit visibleRect; + + // Default matches image unless visibleRect is provided. + [EnforceRange] unsigned long displayWidth; + [EnforceRange] unsigned long displayHeight; +}; + +dictionary VideoFrameBufferInit { + required VideoPixelFormat format; + required [EnforceRange] unsigned long codedWidth; + required [EnforceRange] unsigned long codedHeight; + required [EnforceRange] long long timestamp; // microseconds + [EnforceRange] unsigned long long duration; // microseconds + + // Default layout is tightly-packed. + sequence layout; + + // Default visible rect is coded size positioned at (0,0) + DOMRectInit visibleRect; + + // Default display dimensions match visibleRect. + [EnforceRange] unsigned long displayWidth; + [EnforceRange] unsigned long displayHeight; + + VideoColorSpaceInit colorSpace; +}; + +dictionary VideoFrameCopyToOptions { + DOMRectInit rect; + sequence layout; +}; + +dictionary PlaneLayout { + // TODO: https://github.com/w3c/webcodecs/pull/488 + required [EnforceRange] unsigned long offset; + required [EnforceRange] unsigned long stride; +}; + +enum VideoPixelFormat { + // 4:2:0 Y, U, V + "I420", + // 4:2:0 Y, U, V, A + "I420A", + // 4:2:2 Y, U, V + "I422", + // 4:4:4 Y, U, V + "I444", + // 4:2:0 Y, UV + "NV12", + // 32bpp RGBA + "RGBA", + // 32bpp RGBX (opaque) + "RGBX", + // 32bpp BGRA + "BGRA", + // 32bpp BGRX (opaque) + "BGRX", +}; diff --git a/dom/webidl/moz.build b/dom/webidl/moz.build index 1e15125188c2..c71e33f293b9 100644 --- a/dom/webidl/moz.build +++ b/dom/webidl/moz.build @@ -976,6 +976,7 @@ WEBIDL_FILES = [ "URLSearchParams.webidl", "ValidityState.webidl", "VideoColorSpace.webidl", + "VideoFrame.webidl", "VideoPlaybackQuality.webidl", "VideoTrack.webidl", "VideoTrackList.webidl", diff --git a/testing/web-platform/meta/webcodecs/videoFrame-construction.any.js.ini b/testing/web-platform/meta/webcodecs/videoFrame-construction.any.js.ini index 3ca5820c7f57..8e2a880ece59 100644 --- a/testing/web-platform/meta/webcodecs/videoFrame-construction.any.js.ini +++ b/testing/web-platform/meta/webcodecs/videoFrame-construction.any.js.ini @@ -1,4 +1,6 @@ [videoFrame-construction.any.worker.html] + prefs: [dom.media.webcodecs.enabled:true] + [Test we can construct a VideoFrame.] expected: FAIL @@ -88,6 +90,8 @@ [videoFrame-construction.any.html] + prefs: [dom.media.webcodecs.enabled:true] + [Test we can construct a VideoFrame.] expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/videoFrame-construction.crossOriginIsolated.https.any.js.ini b/testing/web-platform/meta/webcodecs/videoFrame-construction.crossOriginIsolated.https.any.js.ini index 396b80c5be00..f8ee4c588697 100644 --- a/testing/web-platform/meta/webcodecs/videoFrame-construction.crossOriginIsolated.https.any.js.ini +++ b/testing/web-platform/meta/webcodecs/videoFrame-construction.crossOriginIsolated.https.any.js.ini @@ -1,4 +1,5 @@ [videoFrame-construction.crossOriginIsolated.https.any.worker.html] + prefs: [dom.media.webcodecs.enabled:true] expected: if (os == "android") and fission: [OK, TIMEOUT] [Test SharedArrayBuffer constructed I420 VideoFrame] @@ -9,6 +10,7 @@ [videoFrame-construction.crossOriginIsolated.https.any.html] + prefs: [dom.media.webcodecs.enabled:true] expected: if (os == "android") and debug and not fission and not swgl: [OK, TIMEOUT] [Test SharedArrayBuffer constructed I420 VideoFrame] diff --git a/testing/web-platform/meta/webcodecs/videoFrame-construction.window.js.ini b/testing/web-platform/meta/webcodecs/videoFrame-construction.window.js.ini index 46d7c1ae03e3..91ff52e9991d 100644 --- a/testing/web-platform/meta/webcodecs/videoFrame-construction.window.js.ini +++ b/testing/web-platform/meta/webcodecs/videoFrame-construction.window.js.ini @@ -1,4 +1,5 @@ [videoFrame-construction.window.html] + prefs: [dom.media.webcodecs.enabled:true] expected: if (os == "android") and fission: [OK, TIMEOUT] [Test that timestamp is required when constructing VideoFrame from HTMLImageElement] diff --git a/testing/web-platform/meta/webcodecs/videoFrame-copyTo.any.js.ini b/testing/web-platform/meta/webcodecs/videoFrame-copyTo.any.js.ini index 38572e5df040..a2a99a44081e 100644 --- a/testing/web-platform/meta/webcodecs/videoFrame-copyTo.any.js.ini +++ b/testing/web-platform/meta/webcodecs/videoFrame-copyTo.any.js.ini @@ -1,4 +1,6 @@ [videoFrame-copyTo.any.html] + prefs: [dom.media.webcodecs.enabled:true] + [Test RGBA frame.] expected: FAIL @@ -46,6 +48,8 @@ [videoFrame-copyTo.any.worker.html] + prefs: [dom.media.webcodecs.enabled:true] + [Test RGBA frame.] expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.js.ini b/testing/web-platform/meta/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.js.ini index d1fbd14641fa..ae3c98753722 100644 --- a/testing/web-platform/meta/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.js.ini +++ b/testing/web-platform/meta/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.js.ini @@ -1,4 +1,5 @@ [videoFrame-copyTo.crossOriginIsolated.https.any.html] + prefs: [dom.media.webcodecs.enabled:true] [Test copying I420 frame to SharedArrayBuffer.] expected: FAIL @@ -7,6 +8,7 @@ [videoFrame-copyTo.crossOriginIsolated.https.any.worker.html] + prefs: [dom.media.webcodecs.enabled:true] expected: if (os == "android") and fission: [OK, TIMEOUT] [Test copying I420 frame to SharedArrayBuffer.]