nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1765187 Part 1: Track color primaries independently from coefficients. r=media-playback-reviewers,jgilbert,alwu

Browse source 
This patch attempts to read color primary information from platform
agnostic decoders. It doesn't use any of this information to change how
the video frame is displayed.

It also cleans up some VPX transfer characteristics, to ensure they are
actually retrieved from the codec information, when available.

Differential Revision: https://phabricator.services.mozilla.com/D156362
This commit is contained in:
Brad Werth 2022-10-10 20:12:58 +00:00
parent bdb516e93f
commit 949fa28f4f
19 changed files with 156 additions and 11 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

1
dom/media/MediaData.cpp
View file

@ -280,6 +280,7 @@ PlanarYCbCrData ConstructPlanarYCbCrData(const VideoInfo& aInfo,
data.mPictureRect = aPicture;
data.mStereoMode = aInfo.mStereoMode;
data.mYUVColorSpace = aBuffer.mYUVColorSpace;
data.mColorPrimaries = aBuffer.mColorPrimaries;
data.mColorDepth = aBuffer.mColorDepth;
if (aInfo.mTransferFunction) {
data.mTransferFunction = *aInfo.mTransferFunction;

2
dom/media/MediaData.h
View file

@ -421,6 +421,7 @@ class VideoData : public MediaData {
typedef gfx::ColorDepth ColorDepth;
typedef gfx::ColorRange ColorRange;
typedef gfx::YUVColorSpace YUVColorSpace;
typedef gfx::ColorSpace2 ColorSpace2;
typedef gfx::ChromaSubsampling ChromaSubsampling;
typedef layers::ImageContainer ImageContainer;
typedef layers::Image Image;
@ -444,6 +445,7 @@ class VideoData : public MediaData {
Plane mPlanes[3];
YUVColorSpace mYUVColorSpace = YUVColorSpace::Identity;
ColorSpace2 mColorPrimaries = ColorSpace2::UNKNOWN;
ColorDepth mColorDepth = ColorDepth::COLOR_8;
ColorRange mColorRange = ColorRange::LIMITED;
ChromaSubsampling mChromaSubsampling = ChromaSubsampling::FULL;

3
dom/media/MediaInfo.h
View file

@ -433,7 +433,8 @@ class VideoInfo : public TrackInfo {
// Matrix coefficients (if specified by the video) imply a colorspace.
Maybe<gfx::YUVColorSpace> mColorSpace;
// Color primaries are assumed to match the colorspace.
// Color primaries are independent from the coefficients.
Maybe<gfx::ColorSpace2> mColorPrimaries;
// Transfer functions get their own member, which may not be strongly
// correlated to the colorspace.

2
dom/media/ipc/MediaIPCUtils.h
View file

@ -33,6 +33,7 @@ struct ParamTraits<mozilla::VideoInfo> {
WriteParam(aWriter, aParam.mRotation);
WriteParam(aWriter, aParam.mColorDepth);
WriteParam(aWriter, aParam.mColorSpace);
WriteParam(aWriter, aParam.mColorPrimaries);
WriteParam(aWriter, aParam.mTransferFunction);
WriteParam(aWriter, aParam.mColorRange);
WriteParam(aWriter, aParam.HasAlpha());
@ -51,6 +52,7 @@ struct ParamTraits<mozilla::VideoInfo> {
ReadParam(aReader, &aResult->mRotation) &&
ReadParam(aReader, &aResult->mColorDepth) &&
ReadParam(aReader, &aResult->mColorSpace) &&
ReadParam(aReader, &aResult->mColorPrimaries) &&
ReadParam(aReader, &aResult->mTransferFunction) &&
ReadParam(aReader, &aResult->mColorRange) &&
ReadParam(aReader, &alphaPresent)) {

12
dom/media/platforms/agnostic/AOMDecoder.cpp
View file

@ -220,6 +220,18 @@ RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::ProcessDecode(
b.mColorRange = img->range == AOM_CR_FULL_RANGE ? ColorRange::FULL
: ColorRange::LIMITED;
switch (img->cp) {
case AOM_CICP_CP_BT_709:
b.mColorPrimaries = ColorSpace2::BT709;
break;
case AOM_CICP_CP_BT_2020:
b.mColorPrimaries = ColorSpace2::BT2020;
break;
default:
b.mColorPrimaries = ColorSpace2::BT709;
break;
}
RefPtr<VideoData> v;
v = VideoData::CreateAndCopyData(
mInfo, mImageContainer, aSample->mOffset, aSample->mTime,

1
dom/media/platforms/agnostic/BlankDecoderModule.cpp
View file

@ -69,6 +69,7 @@ already_AddRefed<MediaData> BlankVideoDataCreator::Create(
buffer.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
buffer.mYUVColorSpace = gfx::YUVColorSpace::BT601;
buffer.mColorPrimaries = gfx::ColorSpace2::BT709;
return VideoData::CreateAndCopyData(mInfo, mImageContainer, aSample->mOffset,
aSample->mTime, aSample->mDuration,

19
dom/media/platforms/agnostic/DAV1DDecoder.cpp
View file

@ -230,9 +230,10 @@ int DAV1DDecoder::GetPicture(DecodedData& aData, MediaResult& aResult) {
return 0;
}
// When returning Nothing(), the caller chooses the appropriate default
/* static */ Maybe<gfx::YUVColorSpace> DAV1DDecoder::GetColorSpace(
/* static */
Maybe<gfx::YUVColorSpace> DAV1DDecoder::GetColorSpace(
const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
// When returning Nothing(), the caller chooses the appropriate default.
if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
return Nothing();
}
@ -242,6 +243,18 @@ int DAV1DDecoder::GetPicture(DecodedData& aData, MediaResult& aResult) {
static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
}
/* static */
Maybe<gfx::ColorSpace2> DAV1DDecoder::GetColorPrimaries(
const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
// When returning Nothing(), the caller chooses the appropriate default.
if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
return Nothing();
}
return gfxUtils::CicpToColorPrimaries(
static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
}
already_AddRefed<VideoData> DAV1DDecoder::ConstructImage(
const Dav1dPicture& aPicture) {
VideoData::YCbCrBuffer b;
@ -256,6 +269,8 @@ already_AddRefed<VideoData> DAV1DDecoder::ConstructImage(
b.mYUVColorSpace =
DAV1DDecoder::GetColorSpace(aPicture, sPDMLog)
.valueOr(DefaultColorSpace({aPicture.p.w, aPicture.p.h}));
b.mColorPrimaries = DAV1DDecoder::GetColorPrimaries(aPicture, sPDMLog)
.valueOr(gfx::ColorSpace2::BT709);
b.mColorRange = aPicture.seq_hdr->color_range ? gfx::ColorRange::FULL
: gfx::ColorRange::LIMITED;

3
dom/media/platforms/agnostic/DAV1DDecoder.h
View file

@ -36,6 +36,9 @@ class DAV1DDecoder : public MediaDataDecoder,
static Maybe<gfx::YUVColorSpace> GetColorSpace(const Dav1dPicture& aPicture,
LazyLogModule& aLogger);
static Maybe<gfx::ColorSpace2> GetColorPrimaries(const Dav1dPicture& aPicture,
LazyLogModule& aLogger);
private:
~DAV1DDecoder() = default;
RefPtr<DecodePromise> InvokeDecode(MediaRawData* aSample);

17
dom/media/platforms/agnostic/VPXDecoder.cpp
View file

@ -13,6 +13,7 @@
#include "ImageContainer.h"
#include "TimeUnits.h"
#include "gfx2DGlue.h"
#include "gfxUtils.h"
#include "mozilla/PodOperations.h"
#include "mozilla/SyncRunnable.h"
#include "mozilla/TaskQueue.h"
@ -544,9 +545,9 @@ void VPXDecoder::GetVPCCBox(MediaByteBuffer* aDestBox,
writer.WriteBit(aInfo.mFullRange); // full/restricted range
// See VPXDecoder::VPXStreamInfo enums
writer.WriteU8(2); // colour primaries: unspecified
writer.WriteU8(2); // transfer characteristics: unspecified
writer.WriteU8(2); // matrix coefficients: unspecified
writer.WriteU8(aInfo.mColorPrimaries); // color primaries
writer.WriteU8(aInfo.mTransferFunction); // transfer characteristics
writer.WriteU8(2); // matrix coefficients: unspecified
writer.WriteBits(0,
16); // codecIntializationDataSize (must be 0 for VP8/VP9)
@ -563,6 +564,10 @@ bool VPXDecoder::SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec) {
return false;
}
aDestInfo->mColorPrimaries =
gfxUtils::CicpToColorPrimaries(colorSpace.mPrimaries, sPDMLog);
aDestInfo->mTransferFunction =
gfxUtils::CicpToTransferFunction(colorSpace.mTransfer);
aDestInfo->mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth);
VPXDecoder::SetChroma(info, chroma);
info.mFullRange = colorSpace.mRange == ColorRange::FULL;
@ -604,9 +609,9 @@ void VPXDecoder::ReadVPCCBox(VPXStreamInfo& aDestInfo, MediaByteBuffer* aBox) {
SetChroma(aDestInfo, reader.ReadBits(3));
aDestInfo.mFullRange = reader.ReadBit();
reader.ReadBits(8); // colour primaries
reader.ReadBits(8); // transfer characteristics
reader.ReadBits(8); // matrix coefficients
aDestInfo.mColorPrimaries = reader.ReadBits(8); // color primaries
aDestInfo.mTransferFunction = reader.ReadBits(8); // transfer characteristics
reader.ReadBits(8); // matrix coefficients
MOZ_ASSERT(reader.ReadBits(16) ==
0); // codecInitializationDataSize (must be 0 for VP8/VP9)

31
dom/media/platforms/agnostic/VPXDecoder.h
View file

@ -95,6 +95,37 @@ class VPXDecoder : public MediaDataDecoder,
}
}
uint8_t mColorPrimaries = gfx::CICP::ColourPrimaries::CP_UNSPECIFIED;
gfx::ColorSpace2 ColorPrimaries() const {
switch (mColorPrimaries) {
case gfx::CICP::ColourPrimaries::CP_BT709:
return gfx::ColorSpace2::BT709;
case gfx::CICP::ColourPrimaries::CP_UNSPECIFIED:
return gfx::ColorSpace2::BT709;
case gfx::CICP::ColourPrimaries::CP_BT2020:
return gfx::ColorSpace2::BT2020;
default:
return gfx::ColorSpace2::BT709;
}
}
uint8_t mTransferFunction =
gfx::CICP::TransferCharacteristics::TC_UNSPECIFIED;
gfx::TransferFunction TransferFunction() const {
switch (mTransferFunction) {
case gfx::CICP::TransferCharacteristics::TC_BT709:
return gfx::TransferFunction::BT709;
case gfx::CICP::TransferCharacteristics::TC_SRGB:
return gfx::TransferFunction::SRGB;
case gfx::CICP::TransferCharacteristics::TC_SMPTE2084:
return gfx::TransferFunction::PQ;
case gfx::CICP::TransferCharacteristics::TC_HLG:
return gfx::TransferFunction::HLG;
default:
return gfx::TransferFunction::BT709;
}
}
/*
mFullRange == false then:
For BitDepth equals 8:

17
dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
View file

@ -1017,6 +1017,23 @@ gfx::YUVColorSpace FFmpegVideoDecoder<LIBAV_VER>::GetFrameColorSpace() const {
}
}
gfx::ColorSpace2 FFmpegVideoDecoder<LIBAV_VER>::GetFrameColorPrimaries() const {
AVColorPrimaries colorPrimaries = AVCOL_PRI_UNSPECIFIED;
#if LIBAVCODEC_VERSION_MAJOR > 57
colorPrimaries = mFrame->color_primaries;
#endif
switch (colorPrimaries) {
#if LIBAVCODEC_VERSION_MAJOR >= 55
case AVCOL_PRI_BT2020:
return gfx::ColorSpace2::BT2020;
#endif
case AVCOL_PRI_BT709:
return gfx::ColorSpace2::BT709;
default:
return gfx::ColorSpace2::BT709;
}
}
gfx::ColorRange FFmpegVideoDecoder<LIBAV_VER>::GetFrameColorRange() const {
AVColorRange range = AVCOL_RANGE_UNSPECIFIED;
#if LIBAVCODEC_VERSION_MAJOR > 58

1
dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
View file

@ -98,6 +98,7 @@ class FFmpegVideoDecoder<LIBAV_VER>
#endif
}
gfx::YUVColorSpace GetFrameColorSpace() const;
gfx::ColorSpace2 GetFrameColorPrimaries() const;
gfx::ColorRange GetFrameColorRange() const;
MediaResult CreateImage(int64_t aOffset, int64_t aPts, int64_t aDuration,

6
dom/media/platforms/omx/OmxDataDecoder.cpp
View file

@ -951,6 +951,12 @@ already_AddRefed<VideoData> MediaDataHelper::CreateYUV420VideoData(
}
b.mYUVColorSpace = *maybeColorSpace;
auto maybeColorPrimaries = info.mColorPrimaries;
if (!maybeColorPrimaries) {
maybeColorPrimaries = Some(gfx::ColorSpace2::BT709);
}
b.mColorPrimaries = *maybeColorPrimaries;
RefPtr<VideoData> data = VideoData::CreateAndCopyData(
info, mImageContainer,
0, // Filled later by caller.

8
dom/media/platforms/wrappers/MediaChangeMonitor.cpp
View file

@ -142,6 +142,11 @@ class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
mCurrentConfig.mDisplay.height = spsdata.display_height;
mCurrentConfig.mColorDepth = spsdata.ColorDepth();
mCurrentConfig.mColorSpace = Some(spsdata.ColorSpace());
// spsdata.colour_primaries has the same values as
// gfx::CICP::ColourPrimaries.
mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
static_cast<gfx::CICP::ColourPrimaries>(spsdata.colour_primaries),
gMediaDecoderLog);
// spsdata.transfer_characteristics has the same values as
// gfx::CICP::TransferCharacteristics.
mCurrentConfig.mTransferFunction = gfxUtils::CicpToTransferFunction(
@ -272,6 +277,7 @@ class VPXChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
mCurrentConfig.mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth);
mCurrentConfig.mColorSpace = Some(info.ColorSpace());
// VPX bitstream doesn't specify color primaries.
// We don't update the transfer function here, because VPX bitstream
// doesn't specify the transfer function. Instead, we keep the transfer
@ -347,6 +353,8 @@ class AV1ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
mCurrentConfig.mColorSpace = gfxUtils::CicpToColorSpace(
aInfo.mColorSpace.mMatrix, aInfo.mColorSpace.mPrimaries,
gMediaDecoderLog);
mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
aInfo.mColorSpace.mPrimaries, gMediaDecoderLog);
mCurrentConfig.mTransferFunction =
gfxUtils::CicpToTransferFunction(aInfo.mColorSpace.mTransfer);
mCurrentConfig.mColorRange = aInfo.mColorSpace.mRange;

4
dom/media/webm/WebMDemuxer.cpp
View file

@ -307,6 +307,10 @@ nsresult WebMDemuxer::ReadMetadata() {
return NS_ERROR_FAILURE;
}
mInfo.mVideo.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
static_cast<gfx::CICP::ColourPrimaries>(params.primaries),
gMediaDemuxerLog);
// For VPX, this is our only chance to capture the transfer
// characteristics, which we can't get from a VPX bitstream later.
// We only need this value if the video is using the BT2020

1
gfx/layers/ImageContainer.h
View file

@ -669,6 +669,7 @@ struct PlanarYCbCrData {
StereoMode mStereoMode = StereoMode::MONO;
gfx::ColorDepth mColorDepth = gfx::ColorDepth::COLOR_8;
gfx::YUVColorSpace mYUVColorSpace = gfx::YUVColorSpace::Default;
gfx::ColorSpace2 mColorPrimaries = gfx::ColorSpace2::UNKNOWN;
gfx::TransferFunction mTransferFunction = gfx::TransferFunction::BT709;
gfx::ColorRange mColorRange = gfx::ColorRange::LIMITED;
gfx::ChromaSubsampling mChromaSubsampling = gfx::ChromaSubsampling::FULL;

23
gfx/thebes/gfxUtils.cpp
View file

@ -291,8 +291,8 @@ static already_AddRefed<gfxDrawable> CreateSamplingRestrictedDrawable(
DrawTarget* destDrawTarget = aContext->GetDrawTarget();
// We've been not using CreateSamplingRestrictedDrawable in a bunch of places
// for a while. Let's disable it everywhere and confirm that it's ok to get rid
// of.
// for a while. Let's disable it everywhere and confirm that it's ok to get
// rid of.
if (destDrawTarget->GetBackendType() == BackendType::DIRECT2D1_1 || (true)) {
return nullptr;
}
@ -1385,6 +1385,25 @@ const float kIdentityNarrowYCbCrToRGB_RowMajor[16] = {
}
}
// Translate from CICP values to the color primaries we support, or return
// Nothing() if there is no appropriate match to let the caller choose
// a default or generate an error.
//
// See Rec. ITU-T H.273 (12/2016) for details on CICP
/* static */ Maybe<gfx::ColorSpace2> gfxUtils::CicpToColorPrimaries(
const CICP::ColourPrimaries aColourPrimaries, LazyLogModule& aLogger) {
switch (aColourPrimaries) {
case CICP::ColourPrimaries::CP_BT709:
return Some(gfx::ColorSpace2::BT709);
case CICP::ColourPrimaries::CP_BT2020:
return Some(gfx::ColorSpace2::BT2020);
default:
MOZ_LOG(aLogger, LogLevel::Debug,
("Unsupported color primaries value: %hhu", aColourPrimaries));
return {};
}
}
// Translate from CICP values to the transfer functions we support, or return
// Nothing() if there is no appropriate match.
//

4
gfx/thebes/gfxUtils.h
View file

@ -232,6 +232,10 @@ class gfxUtils {
const mozilla::gfx::CICP::ColourPrimaries,
mozilla::LazyLogModule& aLogger);
static mozilla::Maybe<mozilla::gfx::ColorSpace2> CicpToColorPrimaries(
const mozilla::gfx::CICP::ColourPrimaries,
mozilla::LazyLogModule& aLogger);
static mozilla::Maybe<mozilla::gfx::TransferFunction> CicpToTransferFunction(
const mozilla::gfx::CICP::TransferCharacteristics);

12
image/decoders/nsAVIFDecoder.cpp
View file

@ -1018,6 +1018,12 @@ AVIFDecodedData Dav1dDecoder::Dav1dPictureToDecodedData(
seq_hdr.color_range ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED;
data.mColorRange = GetAVIFColorRange(aNclx, av1ColorRange);
auto colorPrimaries =
gfxUtils::CicpToColorPrimaries(data.mColourPrimaries, sAVIFLog);
if (colorPrimaries.isSome()) {
data.mColorPrimaries = *colorPrimaries;
}
if (aAlphaPlane) {
MOZ_ASSERT(aAlphaPlane->stride[0] == data.mYStride);
data.mAlpha.emplace();
@ -1100,6 +1106,12 @@ AVIFDecodedData AOMDecoder::AOMImageToToDecodedData(
data.SetCicpValues(aNclx, av1ColourPrimaries, av1TransferCharacteristics,
av1MatrixCoefficients);
auto colorPrimaries =
gfxUtils::CicpToColorPrimaries(data.mColourPrimaries, sAVIFLog);
if (colorPrimaries.isSome()) {
data.mColorPrimaries = *colorPrimaries;
}
if (aAlphaPlane) {
MOZ_ASSERT(aAlphaPlane->stride[AOM_PLANE_Y] == data.mYStride);
data.mAlpha.emplace();