mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-11-04 10:18:41 +02:00
d69995a9b5
This patch adds a mechanism to close VideoFrame if Close() is never called. The image held in the VideoFrame has to be release so there won't be any memory leak. If the image held in the VideoFrame is Close()d by the users, then no auto-release will be executed. If it's never called, then the added code will release the resource in one of the following cases: 1. The VideoFrame is cycle-collected 2. The VideoFrame's refcount is down to zero 3. The worker is going away (if VideoFrame is created in worker) 4. The process is shutting down Differential Revision: https://phabricator.services.mozilla.com/D199181
266 lines
9.4 KiB
C++
266 lines
9.4 KiB
C++
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
/* vim:set ts=2 sw=2 sts=2 et cindent: */
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#ifndef mozilla_dom_VideoFrame_h
|
|
#define mozilla_dom_VideoFrame_h
|
|
|
|
#include "js/TypeDecls.h"
|
|
#include "mozilla/Attributes.h"
|
|
#include "mozilla/ErrorResult.h"
|
|
#include "mozilla/NotNull.h"
|
|
#include "mozilla/Span.h"
|
|
#include "mozilla/dom/BindingDeclarations.h"
|
|
#include "mozilla/dom/TypedArray.h"
|
|
#include "mozilla/dom/VideoColorSpaceBinding.h"
|
|
#include "mozilla/dom/WorkerRef.h"
|
|
#include "mozilla/gfx/Point.h"
|
|
#include "mozilla/gfx/Rect.h"
|
|
#include "mozilla/media/MediaUtils.h"
|
|
#include "nsCycleCollectionParticipant.h"
|
|
#include "nsWrapperCache.h"
|
|
|
|
class nsIGlobalObject;
|
|
class nsIURI;
|
|
|
|
namespace mozilla {
|
|
|
|
namespace layers {
|
|
class Image;
|
|
} // namespace layers
|
|
|
|
namespace dom {
|
|
|
|
class DOMRectReadOnly;
|
|
class HTMLCanvasElement;
|
|
class HTMLImageElement;
|
|
class HTMLVideoElement;
|
|
class ImageBitmap;
|
|
class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
|
|
class OffscreenCanvas;
|
|
class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
|
|
class Promise;
|
|
class SVGImageElement;
|
|
class StructuredCloneHolder;
|
|
class VideoColorSpace;
|
|
class VideoFrame;
|
|
enum class VideoPixelFormat : uint8_t;
|
|
struct VideoFrameBufferInit;
|
|
struct VideoFrameInit;
|
|
struct VideoFrameCopyToOptions;
|
|
|
|
} // namespace dom
|
|
} // namespace mozilla
|
|
|
|
namespace mozilla::dom {
|
|
|
|
struct VideoFrameData {
|
|
VideoFrameData(layers::Image* aImage, const Maybe<VideoPixelFormat>& aFormat,
|
|
gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
|
|
Maybe<uint64_t> aDuration, int64_t aTimestamp,
|
|
const VideoColorSpaceInit& aColorSpace);
|
|
VideoFrameData(const VideoFrameData& aData) = default;
|
|
|
|
const RefPtr<layers::Image> mImage;
|
|
const Maybe<VideoPixelFormat> mFormat;
|
|
const gfx::IntRect mVisibleRect;
|
|
const gfx::IntSize mDisplaySize;
|
|
const Maybe<uint64_t> mDuration;
|
|
const int64_t mTimestamp;
|
|
const VideoColorSpaceInit mColorSpace;
|
|
};
|
|
|
|
struct VideoFrameSerializedData : VideoFrameData {
|
|
VideoFrameSerializedData(const VideoFrameData& aData,
|
|
gfx::IntSize aCodedSize);
|
|
|
|
const gfx::IntSize mCodedSize;
|
|
};
|
|
|
|
class VideoFrame final : public nsISupports, public nsWrapperCache {
|
|
public:
|
|
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
|
|
NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame)
|
|
|
|
public:
|
|
VideoFrame(nsIGlobalObject* aParent, const RefPtr<layers::Image>& aImage,
|
|
const Maybe<VideoPixelFormat>& aFormat, gfx::IntSize aCodedSize,
|
|
gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
|
|
const Maybe<uint64_t>& aDuration, int64_t aTimestamp,
|
|
const VideoColorSpaceInit& aColorSpace);
|
|
VideoFrame(nsIGlobalObject* aParent, const VideoFrameSerializedData& aData);
|
|
VideoFrame(const VideoFrame& aOther);
|
|
|
|
protected:
|
|
~VideoFrame();
|
|
|
|
public:
|
|
nsIGlobalObject* GetParentObject() const;
|
|
|
|
JSObject* WrapObject(JSContext* aCx,
|
|
JS::Handle<JSObject*> aGivenProto) override;
|
|
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, HTMLImageElement& aImageElement,
|
|
const VideoFrameInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement,
|
|
const VideoFrameInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement,
|
|
const VideoFrameInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement,
|
|
const VideoFrameInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas,
|
|
const VideoFrameInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
|
|
ImageBitmap& aImageBitmap,
|
|
const VideoFrameInit& aInit,
|
|
ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
|
|
VideoFrame& aVideoFrame,
|
|
const VideoFrameInit& aInit,
|
|
ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, const ArrayBufferView& aBufferView,
|
|
const VideoFrameBufferInit& aInit, ErrorResult& aRv);
|
|
static already_AddRefed<VideoFrame> Constructor(
|
|
const GlobalObject& aGlobal, const ArrayBuffer& aBuffer,
|
|
const VideoFrameBufferInit& aInit, ErrorResult& aRv);
|
|
|
|
Nullable<VideoPixelFormat> GetFormat() const;
|
|
|
|
uint32_t CodedWidth() const;
|
|
|
|
uint32_t CodedHeight() const;
|
|
|
|
already_AddRefed<DOMRectReadOnly> GetCodedRect() const;
|
|
|
|
already_AddRefed<DOMRectReadOnly> GetVisibleRect() const;
|
|
|
|
uint32_t DisplayWidth() const;
|
|
|
|
uint32_t DisplayHeight() const;
|
|
|
|
Nullable<uint64_t> GetDuration() const;
|
|
|
|
int64_t Timestamp() const;
|
|
|
|
already_AddRefed<VideoColorSpace> ColorSpace() const;
|
|
|
|
uint32_t AllocationSize(const VideoFrameCopyToOptions& aOptions,
|
|
ErrorResult& aRv);
|
|
|
|
already_AddRefed<Promise> CopyTo(
|
|
const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
|
|
const VideoFrameCopyToOptions& aOptions, ErrorResult& aRv);
|
|
|
|
already_AddRefed<VideoFrame> Clone(ErrorResult& aRv) const;
|
|
|
|
void Close();
|
|
bool IsClosed() const;
|
|
|
|
// [Serializable] implementations: {Read, Write}StructuredClone
|
|
static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
|
|
JSStructuredCloneReader* aReader,
|
|
const VideoFrameSerializedData& aData);
|
|
|
|
bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
|
|
StructuredCloneHolder* aHolder) const;
|
|
|
|
// [Transferable] implementations: Transfer, FromTransferred
|
|
using TransferredData = VideoFrameSerializedData;
|
|
|
|
UniquePtr<TransferredData> Transfer();
|
|
|
|
static already_AddRefed<VideoFrame> FromTransferred(nsIGlobalObject* aGlobal,
|
|
TransferredData* aData);
|
|
|
|
// Native only methods.
|
|
const gfx::IntSize& NativeCodedSize() const { return mCodedSize; }
|
|
const gfx::IntSize& NativeDisplaySize() const { return mDisplaySize; }
|
|
const gfx::IntRect& NativeVisibleRect() const { return mVisibleRect; }
|
|
already_AddRefed<layers::Image> GetImage() const;
|
|
|
|
nsCString ToString() const;
|
|
|
|
public:
|
|
// A VideoPixelFormat wrapper providing utilities for VideoFrame.
|
|
class Format final {
|
|
public:
|
|
explicit Format(const VideoPixelFormat& aFormat);
|
|
~Format() = default;
|
|
const VideoPixelFormat& PixelFormat() const;
|
|
gfx::SurfaceFormat ToSurfaceFormat() const;
|
|
void MakeOpaque();
|
|
|
|
// TODO: Assign unique value for each plane?
|
|
// The value indicates the order of the plane in format.
|
|
enum class Plane : uint8_t { Y = 0, RGBA = Y, U = 1, UV = U, V = 2, A = 3 };
|
|
nsTArray<Plane> Planes() const;
|
|
const char* PlaneName(const Plane& aPlane) const;
|
|
uint32_t SampleBytes(const Plane& aPlane) const;
|
|
gfx::IntSize SampleSize(const Plane& aPlane) const;
|
|
bool IsValidSize(const gfx::IntSize& aSize) const;
|
|
size_t SampleCount(const gfx::IntSize& aSize) const;
|
|
|
|
private:
|
|
bool IsYUV() const;
|
|
VideoPixelFormat mFormat;
|
|
};
|
|
|
|
private:
|
|
// VideoFrame can run on either main thread or worker thread.
|
|
void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(VideoFrame); }
|
|
|
|
VideoFrameData GetVideoFrameData() const;
|
|
|
|
// Below helpers are used to automatically release the holding Resource if
|
|
// VideoFrame is never Close()d by the users.
|
|
void StartAutoClose();
|
|
void StopAutoClose();
|
|
void CloseIfNeeded();
|
|
|
|
// A class representing the VideoFrame's data.
|
|
class Resource final {
|
|
public:
|
|
Resource(const RefPtr<layers::Image>& aImage, Maybe<Format>&& aFormat);
|
|
Resource(const Resource& aOther);
|
|
~Resource() = default;
|
|
Maybe<VideoPixelFormat> TryPixelFormat() const;
|
|
uint32_t Stride(const Format::Plane& aPlane) const;
|
|
bool CopyTo(const Format::Plane& aPlane, const gfx::IntRect& aRect,
|
|
Span<uint8_t>&& aPlaneDest, size_t aDestinationStride) const;
|
|
|
|
const RefPtr<layers::Image> mImage;
|
|
// Nothing() if mImage is not in VideoPixelFormat
|
|
const Maybe<Format> mFormat;
|
|
};
|
|
|
|
nsCOMPtr<nsIGlobalObject> mParent;
|
|
|
|
// Use Maybe instead of UniquePtr to allow copy ctor.
|
|
// The mResource's existence is used as the [[Detached]] for [Transferable].
|
|
Maybe<const Resource> mResource; // Nothing() after `Close()`d
|
|
|
|
// TODO: Replace this by mResource->mImage->GetSize()?
|
|
gfx::IntSize mCodedSize;
|
|
gfx::IntRect mVisibleRect;
|
|
gfx::IntSize mDisplaySize;
|
|
|
|
Maybe<uint64_t> mDuration;
|
|
int64_t mTimestamp;
|
|
VideoColorSpaceInit mColorSpace;
|
|
|
|
// The following are used to help monitoring mResource release.
|
|
UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker = nullptr;
|
|
RefPtr<WeakWorkerRef> mWorkerRef = nullptr;
|
|
};
|
|
|
|
} // namespace mozilla::dom
|
|
|
|
#endif // mozilla_dom_VideoFrame_h
|