Skip to content

Commit

Permalink
Don't set codec color info for default SDR
Browse files Browse the repository at this point in the history 
Some media can read color info values from the bitstream
and may partially set some of the SDR default values in
Format.ColorInfo. Setting these default values for SDR can
confuse some codecs and may also prevent adaptive ABR
switches if not all ColorInfo values are set in exactly the
same way.

We can avoid any influence of HDR color info handling by
disabling setting the color info MediaFormat keys for SDR
video and also avoid codec reset at format changes if both
formats are SDR with slightly different ColorInfo settings.

To identify "SDR" ColorInfo instances, we need to do some
fuzzy matching as many of the default values are assumed to
match the SDR profile even if not set.

Issue: androidx#1158
PiperOrigin-RevId: 617473937
  • Loading branch information
@tonihei @copybara-github
tonihei authored and copybara-github committed Mar 20, 2024
1 parent 3272ad5 commit 3a7d31a
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 28 deletions.
3 changes: 3 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
* Add workaround that ensures the first frame is always rendered while
tunneling even if the device does not do this automatically as required
by the API ([#1169](https://github.com/androidx/media/issues/1169)).
* Fix issue where HDR color info handling causes codec mishavior and
prevents adaptive format switches for SDR video tracks
([#1158](https://github.com/androidx/media/issues/1158)).
* Text:
* WebVTT: Prevent directly consecutive cues from creating spurious
additional `CuesWithTiming` instances from `WebvttParser.parse`
Expand Down
42 changes: 37 additions & 5 deletions libraries/common/src/main/java/androidx/media3/common/ColorInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import androidx.media3.common.util.Util;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.Arrays;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.dataflow.qual.Pure;

/**
Expand Down Expand Up @@ -172,6 +173,34 @@ public ColorInfo build() {
.setColorTransfer(C.COLOR_TRANSFER_SRGB)
.build();

/**
* Returns whether the given color info is equivalent to values for a standard dynamic range video
* that could generally be assumed if no further information is given.
*
* <p>The color info is deemed to be equivalent to SDR video if it either has unset values or
* values matching a 8-bit (chroma+luma), BT.709 or BT.601 color space, SDR transfer and Limited
* range color info.
*
* @param colorInfo The color info to evaluate.
* @return Whether the given color info is equivalent to the assumed default SDR color info.
*/
@EnsuresNonNullIf(result = false, expression = "#1")
public static boolean isEquivalentToAssumedSdrDefault(@Nullable ColorInfo colorInfo) {
if (colorInfo == null) {
return true;
}
return (colorInfo.colorSpace == Format.NO_VALUE
|| colorInfo.colorSpace == C.COLOR_SPACE_BT709
|| colorInfo.colorSpace == C.COLOR_SPACE_BT601)
&& (colorInfo.colorRange == Format.NO_VALUE
|| colorInfo.colorRange == C.COLOR_RANGE_LIMITED)
&& (colorInfo.colorTransfer == Format.NO_VALUE
|| colorInfo.colorTransfer == C.COLOR_TRANSFER_SDR)
&& colorInfo.hdrStaticInfo == null
&& (colorInfo.chromaBitdepth == Format.NO_VALUE || colorInfo.chromaBitdepth == 8)
&& (colorInfo.lumaBitdepth == Format.NO_VALUE || colorInfo.lumaBitdepth == 8);
}

/**
* Returns the {@link C.ColorSpace} corresponding to the given ISO color primary code, as per
* table A.7.21.1 in Rec. ITU-T T.832 (03/2009), or {@link Format#NO_VALUE} if no mapping can be
Expand Down Expand Up @@ -232,22 +261,25 @@ public static boolean isTransferHdr(@Nullable ColorInfo colorInfo) {
|| colorInfo.colorTransfer == C.COLOR_TRANSFER_ST2084);
}

/** The {@link C.ColorSpace}. */
/** The {@link C.ColorSpace}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorSpace int colorSpace;

/** The {@link C.ColorRange}. */
/** The {@link C.ColorRange}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorRange int colorRange;

/** The {@link C.ColorTransfer}. */
/** The {@link C.ColorTransfer}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorTransfer int colorTransfer;

/** HdrStaticInfo as defined in CTA-861.3, or null if none specified. */
@Nullable public final byte[] hdrStaticInfo;

/** The bit depth of the luma samples of the video. */
/** The bit depth of the luma samples of the video, or {@link Format#NO_VALUE} if not set. */
public final int lumaBitdepth;

/** The bit depth of the chroma samples of the video. It may differ from the luma bit depth. */
/**
* The bit depth of the chroma samples of the video, or {@link Format#NO_VALUE} if not set. It may
* differ from the luma bit depth.
*/
public final int chromaBitdepth;

// Lazily initialized hashcode.
Expand Down
2 changes: 1 addition & 1 deletion libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ public static void maybeSetByteBuffer(MediaFormat format, String key, @Nullable
*/
@SuppressWarnings("InlinedApi")
public static void maybeSetColorInfo(MediaFormat format, @Nullable ColorInfo colorInfo) {
if (colorInfo != null) {
if (!ColorInfo.isEquivalentToAssumedSdrDefault(colorInfo)) {
maybeSetInteger(format, MediaFormat.KEY_COLOR_TRANSFER, colorInfo.colorTransfer);
maybeSetInteger(format, MediaFormat.KEY_COLOR_STANDARD, colorInfo.colorSpace);
maybeSetInteger(format, MediaFormat.KEY_COLOR_RANGE, colorInfo.colorRange);
Expand Down
14 changes: 14 additions & 0 deletions libraries/common/src/test/java/androidx/media3/common/util/MediaFormatUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,9 +233,23 @@ public void createMediaFormatFromFormat_withPopulatedFormat_generatesExpectedEnt
@Test
public void createMediaFormatFromFormat_withCustomPcmEncoding_setsCustomPcmEncodingEntry() {
Format format = new Format.Builder().setPcmEncoding(C.ENCODING_PCM_16BIT_BIG_ENDIAN).build();

MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);

assertThat(mediaFormat.getInteger(MediaFormatUtil.KEY_PCM_ENCODING_EXTENDED))
.isEqualTo(C.ENCODING_PCM_16BIT_BIG_ENDIAN);
assertThat(mediaFormat.containsKey(MediaFormat.KEY_PCM_ENCODING)).isFalse();
}

@Test
public void createMediaFormatFromFormat_withSdrColorInfo_omitsMediaFormatColorInfoKeys() {
Format format = new Format.Builder().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();

MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);

assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_RANGE)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_STANDARD)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_HDR_STATIC_INFO)).isFalse();
}
}
6 changes: 5 additions & 1 deletion libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Assertions;
Expand Down Expand Up @@ -422,7 +423,10 @@ public DecoderReuseEvaluation canReuseCodec(Format oldFormat, Format newFormat)
&& (oldFormat.width != newFormat.width || oldFormat.height != newFormat.height)) {
discardReasons |= DISCARD_REASON_VIDEO_RESOLUTION_CHANGED;
}
if (!Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) {
if ((!ColorInfo.isEquivalentToAssumedSdrDefault(oldFormat.colorInfo)
|| !ColorInfo.isEquivalentToAssumedSdrDefault(newFormat.colorInfo))
&& !Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) {
// Don't perform detailed checks if both ColorInfos fall within the default SDR assumption.
discardReasons |= DISCARD_REASON_VIDEO_COLOR_INFO_CHANGED;
}
if (needsAdaptationReconfigureWorkaround(name)
Expand Down
96 changes: 75 additions & 21 deletions ...ries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public final class MediaCodecInfoTest {
.build();

@Test
public void canKeepCodec_withDifferentMimeType_returnsNo() {
public void canReuseCodec_withDifferentMimeType_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);

Format hdAv1Format = FORMAT_H264_HD.buildUpon().setSampleMimeType(VIDEO_AV1).build();
Expand All @@ -89,7 +89,7 @@ public void canKeepCodec_withDifferentMimeType_returnsNo() {
}

@Test
public void canKeepCodec_withRotation_returnsNo() {
public void canReuseCodec_withRotation_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);

Format hdRotatedFormat = FORMAT_H264_HD.buildUpon().setRotationDegrees(90).build();
Expand All @@ -104,7 +104,7 @@ public void canKeepCodec_withRotation_returnsNo() {
}

@Test
public void canKeepCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() {
public void canReuseCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);

assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K))
Expand All @@ -118,7 +118,7 @@ public void canKeepCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconf
}

@Test
public void canKeepCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() {
public void canReuseCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K))
Expand All @@ -132,7 +132,7 @@ public void canKeepCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() {
}

@Test
public void canKeepCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() {
public void canReuseCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdVariantFormat =
Expand All @@ -148,11 +148,11 @@ public void canKeepCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithRecon
}

@Test
public void canKeepCodec_colorInfoOmittedFromNewFormat_returnsNo() {
public void canReuseCodec_hdrToSdr_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(codecInfo.canReuseCodec(hdrVariantFormat, FORMAT_H264_4K))
.isEqualTo(
new DecoderReuseEvaluation(
Expand All @@ -164,11 +164,11 @@ public void canKeepCodec_colorInfoOmittedFromNewFormat_returnsNo() {
}

@Test
public void canKeepCodec_colorInfoOmittedFromOldFormat_returnsNo() {
public void canReuseCodec_sdrToHdr_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, hdrVariantFormat))
.isEqualTo(
new DecoderReuseEvaluation(
Expand All @@ -180,13 +180,13 @@ public void canKeepCodec_colorInfoOmittedFromOldFormat_returnsNo() {
}

@Test
public void canKeepCodec_colorInfoChange_returnsNo() {
public void canReuseCodec_hdrColorInfoChange_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat1 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
Format hdrVariantFormat2 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build();
assertThat(codecInfo.canReuseCodec(hdrVariantFormat1, hdrVariantFormat2))
.isEqualTo(
new DecoderReuseEvaluation(
Expand All @@ -198,7 +198,61 @@ public void canKeepCodec_colorInfoChange_returnsNo() {
}

@Test
public void canKeepCodec_audioWithDifferentChannelCounts_returnsNo() {
public void canReuseCodec_nullColorInfoToSdr_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format variantWithColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, variantWithColorInfo))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
FORMAT_H264_4K,
variantWithColorInfo,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}

@Test
public void canReuseCodec_sdrToNullColorInfo_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format variantWithColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
assertThat(codecInfo.canReuseCodec(variantWithColorInfo, FORMAT_H264_4K))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
variantWithColorInfo,
FORMAT_H264_4K,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}

@Test
public void canReuseCodec_sdrToSdrWithPartialInformation_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format variantWithFullColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
Format variantWithPartialColorInfo =
FORMAT_H264_4K
.buildUpon()
.setColorInfo(
ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(Format.NO_VALUE).build())
.build();
assertThat(codecInfo.canReuseCodec(variantWithFullColorInfo, variantWithPartialColorInfo))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
variantWithFullColorInfo,
variantWithPartialColorInfo,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}

@Test
public void canReuseCodec_audioWithDifferentChannelCounts_returnsNo() {
MediaCodecInfo codecInfo = buildAacCodecInfo();

assertThat(codecInfo.canReuseCodec(FORMAT_AAC_STEREO, FORMAT_AAC_SURROUND))
Expand All @@ -212,7 +266,7 @@ public void canKeepCodec_audioWithDifferentChannelCounts_returnsNo() {
}

@Test
public void canKeepCodec_audioWithSameChannelCounts_returnsYesWithFlush() {
public void canReuseCodec_audioWithSameChannelCounts_returnsYesWithFlush() {
MediaCodecInfo codecInfo = buildAacCodecInfo();

Format stereoVariantFormat = FORMAT_AAC_STEREO.buildUpon().setAverageBitrate(100).build();
Expand All @@ -227,7 +281,7 @@ public void canKeepCodec_audioWithSameChannelCounts_returnsYesWithFlush() {
}

@Test
public void canKeepCodec_audioWithDifferentInitializationData_returnsNo() {
public void canReuseCodec_audioWithDifferentInitializationData_returnsNo() {
MediaCodecInfo codecInfo = buildAacCodecInfo();

Format stereoVariantFormat =
Expand Down Expand Up @@ -310,7 +364,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromCompleteNewFormat_
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ true))
Expand All @@ -323,7 +377,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromIncompleteNewForma
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ false))
Expand All @@ -336,7 +390,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromOldFormat_returnsF
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
FORMAT_H264_4K, hdrVariantFormat, /* isNewFormatComplete= */ true))
Expand All @@ -349,9 +403,9 @@ public void isSeamlessAdaptationSupported_colorInfoChange_returnsFalse() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);

Format hdrVariantFormat1 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
Format hdrVariantFormat2 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat1, hdrVariantFormat2, /* isNewFormatComplete= */ true))
Expand Down Expand Up @@ -429,7 +483,7 @@ private static MediaCodecInfo buildAacCodecInfo() {
/* secure= */ false);
}

private static ColorInfo buildColorInfo(@C.ColorSpace int colorSpace) {
private static ColorInfo buildHdrColorInfo(@C.ColorSpace int colorSpace) {
return new ColorInfo.Builder()
.setColorSpace(colorSpace)
.setColorRange(C.COLOR_RANGE_FULL)
Expand Down

0 comments on commit 3a7d31a

Please sign in to comment.