From 4b99db938a7c263d84f13e89c864ca9fdcf387f6 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Thu, 13 Jan 2022 15:16:06 +0530 Subject: [PATCH 1/6] Add support for RTSP H265 Added H265 RTP packet reader and added support for H265 playback through RTSP Change-Id: Ic0e135768bb92cd3343212c2edd84bae6947320e --- .../exoplayer/rtsp/RtpPayloadFormat.java | 4 + .../media3/exoplayer/rtsp/RtspMediaTrack.java | 56 +++- .../DefaultRtpPayloadReaderFactory.java | 2 + .../exoplayer/rtsp/reader/RtpH265Reader.java | 317 ++++++++++++++++++ 4 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 4c4521e682e..297353167b9 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -39,12 +39,14 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_AC3 = "AC3"; private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; private static final String RTP_MEDIA_H264 = "H264"; + private static final String RTP_MEDIA_H265 = "H265"; /** Returns whether the format of a {@link MediaDescription} is supported. */ public static boolean isFormatSupported(MediaDescription mediaDescription) { switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) { case RTP_MEDIA_AC3: case RTP_MEDIA_H264: + case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_GENERIC: return true; default: @@ -65,6 +67,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.AUDIO_AC3; case RTP_MEDIA_H264: return MimeTypes.VIDEO_H264; + case RTP_MEDIA_H265: + return MimeTypes.VIDEO_H265; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 5b6b9a46077..8e2d42135f8 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -44,6 +44,11 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; + private static final String H265_SPS = "sprop-sps"; + private static final String H265_PPS = "sprop-pps"; + private static final String H265_VPS = "sprop-vps"; + private static final String H265_MAX_DON_DIFF = "sprop-max-don-diff"; + /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; /** Prefix for the RFC6381 codecs string for AVC formats. */ @@ -120,6 +125,10 @@ public int hashCode() { checkArgument(!fmtpParameters.isEmpty()); processH264FmtpAttribute(formatBuilder, fmtpParameters); break; + case MimeTypes.VIDEO_H265: + checkArgument(!fmtpParameters.isEmpty()); + processH265FmtpAttribute(formatBuilder, fmtpParameters); + break; case MimeTypes.AUDIO_AC3: // AC3 does not require a FMTP attribute. Fall through. default: @@ -168,8 +177,8 @@ private static void processH264FmtpAttribute( checkArgument(parameterSets.length == 2); ImmutableList initializationData = ImmutableList.of( - getH264InitializationDataFromParameterSet(parameterSets[0]), - getH264InitializationDataFromParameterSet(parameterSets[1])); + getInitializationDataFromParameterSet(parameterSets[0]), + getInitializationDataFromParameterSet(parameterSets[1])); formatBuilder.setInitializationData(initializationData); // Process SPS (Sequence Parameter Set). @@ -191,7 +200,7 @@ private static void processH264FmtpAttribute( } } - private static byte[] getH264InitializationDataFromParameterSet(String parameterSet) { + private static byte[] getInitializationDataFromParameterSet(String parameterSet) { byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT); byte[] decodedParameterNalUnit = new byte[decodedParameterNalData.length + NAL_START_CODE.length]; @@ -210,6 +219,47 @@ private static byte[] getH264InitializationDataFromParameterSet(String parameter return decodedParameterNalUnit; } + private static void processH265FmtpAttribute( + Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { + if (fmtpAttributes.containsKey(H265_MAX_DON_DIFF)) { + checkArgument(Integer.parseInt(checkNotNull(fmtpAttributes.get(H265_MAX_DON_DIFF))) == 0); + } + + checkArgument(fmtpAttributes.containsKey(H265_SPS)); + String spropSPS = checkNotNull(fmtpAttributes.get(H265_SPS)); + checkArgument(fmtpAttributes.containsKey(H265_PPS)); + String spropPPS = checkNotNull(fmtpAttributes.get(H265_PPS)); + checkArgument(fmtpAttributes.containsKey(H265_VPS)); + String spropVPS = checkNotNull(fmtpAttributes.get(H265_VPS)); + String[] parameterSets = new String[] {spropSPS, spropPPS, spropVPS}; + + checkArgument(parameterSets.length == 3); + ImmutableList initializationData = + ImmutableList.of( + getInitializationDataFromParameterSet(parameterSets[0]), + getInitializationDataFromParameterSet(parameterSets[1]), + getInitializationDataFromParameterSet(parameterSets[2])); + formatBuilder.setInitializationData(initializationData); + + // Process SPS (Sequence Parameter Set). + byte[] spsNalDataWithStartCode = initializationData.get(0); + NalUnitUtil.H265SpsData spsData = + NalUnitUtil.parseH265SpsNalUnit( + spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); + formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio); + formatBuilder.setHeight(spsData.height); + formatBuilder.setWidth(spsData.width); + + formatBuilder.setCodecs( + CodecSpecificDataUtil.buildHevcCodecString( + spsData.generalProfileSpace, + spsData.generalTierFlag, + spsData.generalProfileIdc, + spsData.generalProfileCompatibilityFlags, + spsData.constraintBytes, + spsData.generalLevelIdc)); + } + /** * Extracts the track URI. * diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 8fe084c1319..888939b7e89 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -38,6 +38,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { return new RtpAacReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); + case MimeTypes.VIDEO_H265: + return new RtpH265Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java new file mode 100644 index 00000000000..56ccd23382e --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -0,0 +1,317 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.NalUnitUtil; +import androidx.media3.extractor.TrackOutput; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/** + * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * RFC7798 for more details. + */ +/* package */ final class RtpH265Reader implements RtpPayloadReader { + + private static final String TAG = "RtpH265Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + /** Offset of payload data within a FU type A payload. */ + private static final int FU_PAYLOAD_OFFSET = 3; + + /** Single Time Aggregation Packet type A. */ + private static final int RTP_PACKET_TYPE_STAP_A = 48; // RFC7798 Section 4.4.2 + /** Fragmentation Unit type A. */ + private static final int RTP_PACKET_TYPE_FU_A = 49; + + /** IDR NAL unit type. */ + private static final int NAL_IDR_W_LP = 19; + + private static final int NAL_IDR_N_LP = 20; + + /** Scratch for Fragmentation Unit RTP packets. */ + private final ParsableByteArray fuScratchBuffer; + + private final ParsableByteArray nalStartCodeArray = + new ParsableByteArray(NalUnitUtil.NAL_START_CODE); + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + private int previousSequenceNumber; + /** The combined size of a sample that is fragmented into multiple RTP packets. */ + private int fragmentedSampleSizeBytes; + + private long startTimeOffsetUs; + + /** Creates an instance. */ + public RtpH265Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + fuScratchBuffer = new ParsableByteArray(); + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + + int payloadType; + try { + // RFC7798 Section 1.1.4. NAL Unit Header + payloadType = (data.getData()[0] >> 1) & 0x3F; // Type - Bits 1 to 6, inclusive. + } catch (IndexOutOfBoundsException e) { + throw ParserException.createForMalformedManifest(/* message= */ null, e); + } + + checkStateNotNull(trackOutput); + if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_STAP_A) { + processSingleNalUnitPacket(data); + } else if (payloadType == RTP_PACKET_TYPE_STAP_A) { + processSingleTimeAggregationPacket(data); + } else if (payloadType == RTP_PACKET_TYPE_FU_A) { + processFragmentationUnitPacket(data, sequenceNumber); + } else { + throw ParserException.createForMalformedManifest( + String.format("RTP H265 payload type [%d] not supported.", payloadType), + /* cause= */ null); + } + + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata( + timeUs, + bufferFlags, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* encryptionData= */ null); + fragmentedSampleSizeBytes = 0; + } + + previousSequenceNumber = sequenceNumber; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + fragmentedSampleSizeBytes = 0; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + + /** + * Processes Single NAL Unit packet (RFC7798 Section 4.4.1). + * + *

Outputs the single NAL Unit (with start code prepended) to {@link #trackOutput}. Sets {@link + * #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly. + */ + @RequiresNonNull("trackOutput") + private void processSingleNalUnitPacket(ParsableByteArray data) { + // Example of a Single Nal Unit packet + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | PayloadHdr | DONL (conditional) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | | + // | NAL unit payload data | + // | | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | :...OPTIONAL RTP padding | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + int numBytesInData = data.bytesLeft(); + fragmentedSampleSizeBytes += writeStartCode(); + trackOutput.sampleData(data, numBytesInData); + fragmentedSampleSizeBytes += numBytesInData; + + int nalHeaderType = (data.getData()[0] >> 1) & 0x3F; + bufferFlags = getBufferFlagsFromNalType(nalHeaderType); + } + + + /** + * Processes STAP Type A packet (RFC7798 Section 4.4.2). + * + *

Outputs the received aggregation units (with start code prepended) to {@link #trackOutput}. + * Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly. + */ + @RequiresNonNull("trackOutput") + private void processSingleTimeAggregationPacket(ParsableByteArray data) throws ParserException { + // An Example of an AP Packet Containing Two Aggregation + // Units without the DONL and DOND Fields. + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | RTP Header | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | PayloadHdr (Type=48) | NALU 1 Size | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | NALU 1 HDR | | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- NALU 1 Data | + // | | + // | | + // + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | | NALU 2 Size | NALU 2 HDR | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | NALU 2 HDR | | + // +-+-+-+-+-+-+-+- NALU 2 Data | + // | | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | :...OPTIONAL RTP padding | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + throw ParserException.createForMalformedManifest( + "need to implement processSingleTimeAggregationPacket", + /* cause= */ null); + + } + + /** + * Processes Fragmentation Unit Type A packet (RFC7798 Section 4.4.3). + * + *

This method will be invoked multiple times to receive a single frame that is broken down + * into a series of fragmentation units in multiple RTP packets. + * + *

Outputs the received fragmentation units (with start code prepended) to {@link + * #trackOutput}. Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly. + */ + @RequiresNonNull("trackOutput") + private void processFragmentationUnitPacket(ParsableByteArray data, int packetSequenceNumber) { + // Example of a FU packet + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | PayloadHdr (Type=49) | FU header | DONL (cond) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| + // | DONL (cond) | | + // |-+-+-+-+-+-+-+-+ | + // | FU payload | + // | | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | :...OPTIONAL RTP padding | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // FU header + // +---------------+ + // |0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+ + // |S|E| FuType | + // +---------------+ + + int tid = (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header section 1.1.4 + int fuHeader = data.getData()[2]; + int nalUnitType = fuHeader & 0x3F; + byte nalHeader[] = new byte[2]; + + nalHeader[0] = (byte) (nalUnitType << 1); // Section: 1.1.4 + // According to section 1.1.4 in rfc7798, layerId is required to be zero so keeping its value + // zero and copying only tid. + nalHeader[1] = (byte) tid; + boolean isFirstFuPacket = (fuHeader & 0x80) > 0; + boolean isLastFuPacket = (fuHeader & 0x40) > 0; + + if (isFirstFuPacket) { + // Prepends starter code. + fragmentedSampleSizeBytes += writeStartCode(); + + // The bytes needed is 2 (NALU header) + payload size. The original data array has size 3 + // (2 payload + 1 FU header) + payload size. Thus setting the correct header and set position + // to 1. + // Overwrite byte 1 of payload header with byte 0 of HEVC nal header + data.getData()[1] = (byte) nalHeader[0]; + // Overwrite byte FU Header with byte 1 of HEVC nal header + data.getData()[2] = (byte) nalHeader[1]; + fuScratchBuffer.reset(data.getData()); + fuScratchBuffer.setPosition(1); + } else { + // Check that this packet is in the sequence of the previous packet. + int expectedSequenceNumber = (previousSequenceNumber + 1) % RtpPacket.MAX_SEQUENCE_NUMBER; + if (packetSequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." + + " Dropping packet.", + expectedSequenceNumber, packetSequenceNumber)); + return; + } + + // Setting position to ignore payload and FU header. + fuScratchBuffer.reset(data.getData()); + fuScratchBuffer.setPosition(FU_PAYLOAD_OFFSET); + } + + int fragmentSize = fuScratchBuffer.bytesLeft(); + trackOutput.sampleData(fuScratchBuffer, fragmentSize); + fragmentedSampleSizeBytes += fragmentSize; + + if (isLastFuPacket) { + bufferFlags = getBufferFlagsFromNalType(nalUnitType); + } + } + + private int writeStartCode() { + nalStartCodeArray.setPosition(/* position= */ 0); + int bytesWritten = nalStartCodeArray.bytesLeft(); + checkNotNull(trackOutput).sampleData(nalStartCodeArray, bytesWritten); + return bytesWritten; + } + + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + + @C.BufferFlags + private static int getBufferFlagsFromNalType(int nalType) { + return (nalType == NAL_IDR_W_LP || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0; + } +} From 9cb243647f4f83c0ab0c97958def3673f410f231 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Fri, 28 Jan 2022 11:22:59 +0530 Subject: [PATCH 2/6] Clean up RtpH265Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 8e2d42135f8..d7bb6d432a4 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -44,10 +44,10 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; - private static final String H265_SPS = "sprop-sps"; - private static final String H265_PPS = "sprop-pps"; - private static final String H265_VPS = "sprop-vps"; - private static final String H265_MAX_DON_DIFF = "sprop-max-don-diff"; + private static final String PARAMETER_SPROP_H265_SPS = "sprop-sps"; + private static final String PARAMETER_SPROP_H265_PPS = "sprop-pps"; + private static final String PARAMETER_SPROP_H265_VPS = "sprop-vps"; + private static final String PARAMETER_SPROP_H265_MAX_DON_DIFF = "sprop-max-don-diff"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -221,31 +221,30 @@ private static byte[] getInitializationDataFromParameterSet(String parameterSet) private static void processH265FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { - if (fmtpAttributes.containsKey(H265_MAX_DON_DIFF)) { - checkArgument(Integer.parseInt(checkNotNull(fmtpAttributes.get(H265_MAX_DON_DIFF))) == 0); + if (fmtpAttributes.containsKey(PARAMETER_SPROP_H265_MAX_DON_DIFF)) { + checkArgument( + Integer.parseInt(checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_MAX_DON_DIFF))) + == 0, + "non-zero sprop-max-don-diff is not supported"); } - checkArgument(fmtpAttributes.containsKey(H265_SPS)); - String spropSPS = checkNotNull(fmtpAttributes.get(H265_SPS)); - checkArgument(fmtpAttributes.containsKey(H265_PPS)); - String spropPPS = checkNotNull(fmtpAttributes.get(H265_PPS)); - checkArgument(fmtpAttributes.containsKey(H265_VPS)); - String spropVPS = checkNotNull(fmtpAttributes.get(H265_VPS)); - String[] parameterSets = new String[] {spropSPS, spropPPS, spropVPS}; + checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_SPS)); + String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_SPS)); + checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_PPS)); + String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_PPS)); + checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); + String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); - checkArgument(parameterSets.length == 3); - ImmutableList initializationData = - ImmutableList.of( - getInitializationDataFromParameterSet(parameterSets[0]), - getInitializationDataFromParameterSet(parameterSets[1]), - getInitializationDataFromParameterSet(parameterSets[2])); + byte[] vpsNalData = getInitializationDataFromParameterSet(spropVPS); + byte[] spsNalData = getInitializationDataFromParameterSet(spropSPS); + byte[] ppsNalData = getInitializationDataFromParameterSet(spropPPS); + ImmutableList initializationData = ImmutableList.of(vpsNalData, spsNalData, ppsNalData); formatBuilder.setInitializationData(initializationData); // Process SPS (Sequence Parameter Set). - byte[] spsNalDataWithStartCode = initializationData.get(0); NalUnitUtil.H265SpsData spsData = NalUnitUtil.parseH265SpsNalUnit( - spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); + spsNalData, NAL_START_CODE.length, spsNalData.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio); formatBuilder.setHeight(spsData.height); formatBuilder.setWidth(spsData.width); From ace363e1839c010755cefbf5e133367dd6da4370 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Tue, 8 Feb 2022 16:41:45 +0530 Subject: [PATCH 3/6] Fix review comments in RtpH265Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 56 ++++++----- .../exoplayer/rtsp/reader/RtpH265Reader.java | 99 +++++++++---------- 2 files changed, 73 insertions(+), 82 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index d7bb6d432a4..afa3327efbe 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -169,6 +169,26 @@ private static void processAacFmtpAttribute( AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + /** Returns H264/H265 initialization data from RTP parameter set. */ + private static byte[] getInitializationDataFromParameterSet(String parameterSet) { + byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT); + byte[] decodedParameterNalUnit = + new byte[decodedParameterNalData.length + NAL_START_CODE.length]; + System.arraycopy( + NAL_START_CODE, + /* srcPos= */ 0, + decodedParameterNalUnit, + /* destPos= */ 0, + NAL_START_CODE.length); + System.arraycopy( + decodedParameterNalData, + /* srcPos= */ 0, + decodedParameterNalUnit, + /* destPos= */ NAL_START_CODE.length, + decodedParameterNalData.length); + return decodedParameterNalUnit; + } + private static void processH264FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS)); @@ -200,25 +220,6 @@ private static void processH264FmtpAttribute( } } - private static byte[] getInitializationDataFromParameterSet(String parameterSet) { - byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT); - byte[] decodedParameterNalUnit = - new byte[decodedParameterNalData.length + NAL_START_CODE.length]; - System.arraycopy( - NAL_START_CODE, - /* srcPos= */ 0, - decodedParameterNalUnit, - /* destPos= */ 0, - NAL_START_CODE.length); - System.arraycopy( - decodedParameterNalData, - /* srcPos= */ 0, - decodedParameterNalUnit, - /* destPos= */ NAL_START_CODE.length, - decodedParameterNalData.length); - return decodedParameterNalUnit; - } - private static void processH265FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { if (fmtpAttributes.containsKey(PARAMETER_SPROP_H265_MAX_DON_DIFF)) { @@ -228,23 +229,24 @@ private static void processH265FmtpAttribute( "non-zero sprop-max-don-diff is not supported"); } + checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); + String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_SPS)); String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_SPS)); checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_PPS)); String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_PPS)); - checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); - String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); - - byte[] vpsNalData = getInitializationDataFromParameterSet(spropVPS); - byte[] spsNalData = getInitializationDataFromParameterSet(spropSPS); - byte[] ppsNalData = getInitializationDataFromParameterSet(spropPPS); - ImmutableList initializationData = ImmutableList.of(vpsNalData, spsNalData, ppsNalData); + ImmutableList initializationData = + ImmutableList.of( + getInitializationDataFromParameterSet(spropVPS), + getInitializationDataFromParameterSet(spropSPS), + getInitializationDataFromParameterSet(spropPPS)); formatBuilder.setInitializationData(initializationData); // Process SPS (Sequence Parameter Set). + byte[] spsNalDataWithStartCode = initializationData.get(1); NalUnitUtil.H265SpsData spsData = NalUnitUtil.parseH265SpsNalUnit( - spsNalData, NAL_START_CODE.length, spsNalData.length); + spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio); formatBuilder.setHeight(spsData.height); formatBuilder.setWidth(spsData.width); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java index 56ccd23382e..2bc1ee2eaec 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -42,17 +42,28 @@ private static final long MEDIA_CLOCK_FREQUENCY = 90_000; - /** Offset of payload data within a FU type A payload. */ + /** Offset of payload data within a FU payload. */ private static final int FU_PAYLOAD_OFFSET = 3; - /** Single Time Aggregation Packet type A. */ - private static final int RTP_PACKET_TYPE_STAP_A = 48; // RFC7798 Section 4.4.2 - /** Fragmentation Unit type A. */ - private static final int RTP_PACKET_TYPE_FU_A = 49; + /** + * Aggregation Packet. + * + * @see + * RFC7798 Section 4.4.2 + */ + private static final int RTP_PACKET_TYPE_AP = 48; + /** + * Fragmentation Unit. + * + * @see + * RFC7798 Section 4.4.3 + */ + private static final int RTP_PACKET_TYPE_FU = 49; /** IDR NAL unit type. */ - private static final int NAL_IDR_W_LP = 19; - + private static final int NAL_IDR_W_RADL = 19; private static final int NAL_IDR_N_LP = 20; /** Scratch for Fragmentation Unit RTP packets. */ @@ -65,12 +76,10 @@ private @MonotonicNonNull TrackOutput trackOutput; @C.BufferFlags private int bufferFlags; - private long firstReceivedTimestamp; private int previousSequenceNumber; /** The combined size of a sample that is fragmented into multiple RTP packets. */ private int fragmentedSampleSizeBytes; - private long startTimeOffsetUs; /** Creates an instance. */ @@ -84,7 +93,6 @@ public RtpH265Reader(RtpPayloadFormat payloadFormat) { @Override public void createTracks(ExtractorOutput extractorOutput, int trackId) { trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); - castNonNull(trackOutput).format(payloadFormat.format); } @@ -94,7 +102,6 @@ public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} @Override public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) throws ParserException { - int payloadType; try { // RFC7798 Section 1.1.4. NAL Unit Header @@ -104,11 +111,11 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, } checkStateNotNull(trackOutput); - if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_STAP_A) { + if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_AP) { processSingleNalUnitPacket(data); - } else if (payloadType == RTP_PACKET_TYPE_STAP_A) { - processSingleTimeAggregationPacket(data); - } else if (payloadType == RTP_PACKET_TYPE_FU_A) { + } else if (payloadType == RTP_PACKET_TYPE_AP) { + processAggregationPacket(data); + } else if (payloadType == RTP_PACKET_TYPE_FU) { processFragmentationUnitPacket(data, sequenceNumber); } else { throw ParserException.createForMalformedManifest( @@ -173,46 +180,21 @@ private void processSingleNalUnitPacket(ParsableByteArray data) { bufferFlags = getBufferFlagsFromNalType(nalHeaderType); } - /** - * Processes STAP Type A packet (RFC7798 Section 4.4.2). + * Processes an AP packet (RFC7798 Section 4.4.2). * *

Outputs the received aggregation units (with start code prepended) to {@link #trackOutput}. * Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly. */ @RequiresNonNull("trackOutput") - private void processSingleTimeAggregationPacket(ParsableByteArray data) throws ParserException { - // An Example of an AP Packet Containing Two Aggregation - // Units without the DONL and DOND Fields. - // 0 1 2 3 - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | RTP Header | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | PayloadHdr (Type=48) | NALU 1 Size | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | NALU 1 HDR | | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- NALU 1 Data | - // | | - // | | - // + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | | NALU 2 Size | NALU 2 HDR | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | NALU 2 HDR | | - // +-+-+-+-+-+-+-+- NALU 2 Data | - // | | - // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | :...OPTIONAL RTP padding | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - + private void processAggregationPacket(ParsableByteArray data) throws ParserException { throw ParserException.createForMalformedManifest( - "need to implement processSingleTimeAggregationPacket", + "need to implement processAggregationPacket", /* cause= */ null); - } /** - * Processes Fragmentation Unit Type A packet (RFC7798 Section 4.4.3). + * Processes Fragmentation Unit packet (RFC7798 Section 4.4.3). * *

This method will be invoked multiple times to receive a single frame that is broken down * into a series of fragmentation units in multiple RTP packets. @@ -241,15 +223,21 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe // +-+-+-+-+-+-+-+-+ // |S|E| FuType | // +---------------+ - - int tid = (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header section 1.1.4 + // Structure of HEVC NAL unit header + // +---------------+---------------+ + // |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F| Type | LayerId | TID | + // +-------------+-----------------+ + + int tid = + (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header, RFC7798 Section 1.1.4 int fuHeader = data.getData()[2]; int nalUnitType = fuHeader & 0x3F; byte nalHeader[] = new byte[2]; - nalHeader[0] = (byte) (nalUnitType << 1); // Section: 1.1.4 - // According to section 1.1.4 in rfc7798, layerId is required to be zero so keeping its value - // zero and copying only tid. + nalHeader[0] = (byte) (nalUnitType << 1); // RFC7798 Section 1.1.4 + // layerId must be zero according to RFC7798 Section 1.1.4, so copying the tid only nalHeader[1] = (byte) tid; boolean isFirstFuPacket = (fuHeader & 0x80) > 0; boolean isLastFuPacket = (fuHeader & 0x40) > 0; @@ -258,12 +246,13 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe // Prepends starter code. fragmentedSampleSizeBytes += writeStartCode(); - // The bytes needed is 2 (NALU header) + payload size. The original data array has size 3 - // (2 payload + 1 FU header) + payload size. Thus setting the correct header and set position - // to 1. - // Overwrite byte 1 of payload header with byte 0 of HEVC nal header + // Overwrite a few bytes in Rtp buffer to get HEVC NAL unit + // Rtp Byte 0 -> Ignore + // Rtp Byte 1 -> Byte 0 of HEVC NAL header + // Rtp Byte 2 -> Byte 1 of HEVC NAL header + // Rtp Payload -> HEVC NAL bytes, so leave them unchanged + // Set data position from byte 1 as byte 0 was ignored data.getData()[1] = (byte) nalHeader[0]; - // Overwrite byte FU Header with byte 1 of HEVC nal header data.getData()[2] = (byte) nalHeader[1]; fuScratchBuffer.reset(data.getData()); fuScratchBuffer.setPosition(1); @@ -312,6 +301,6 @@ private static long toSampleUs( @C.BufferFlags private static int getBufferFlagsFromNalType(int nalType) { - return (nalType == NAL_IDR_W_LP || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0; + return (nalType == NAL_IDR_W_RADL || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0; } } From 3ef90d9dd4b8e05b87d98e1f03e23572f0c0fec2 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Wed, 9 Feb 2022 15:06:16 +0530 Subject: [PATCH 4/6] Fixed some of nitpick for RtpH265Reader --- .../media3/exoplayer/rtsp/reader/RtpH265Reader.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java index 2bc1ee2eaec..53c3a260d6b 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -62,7 +62,7 @@ */ private static final int RTP_PACKET_TYPE_FU = 49; - /** IDR NAL unit type. */ + /** IDR NAL unit types. */ private static final int NAL_IDR_W_RADL = 19; private static final int NAL_IDR_N_LP = 20; @@ -158,7 +158,7 @@ public void seek(long nextRtpTimestamp, long timeUs) { */ @RequiresNonNull("trackOutput") private void processSingleNalUnitPacket(ParsableByteArray data) { - // Example of a Single Nal Unit packet + // The structure a single NAL unit packet. // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -204,7 +204,7 @@ private void processAggregationPacket(ParsableByteArray data) throws ParserExcep */ @RequiresNonNull("trackOutput") private void processFragmentationUnitPacket(ParsableByteArray data, int packetSequenceNumber) { - // Example of a FU packet + // The structure of an FU packet. // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -246,10 +246,10 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe // Prepends starter code. fragmentedSampleSizeBytes += writeStartCode(); - // Overwrite a few bytes in Rtp buffer to get HEVC NAL unit + // Overwrite a few bytes in Rtp buffer to get HEVC NAL Unit // Rtp Byte 0 -> Ignore - // Rtp Byte 1 -> Byte 0 of HEVC NAL header - // Rtp Byte 2 -> Byte 1 of HEVC NAL header + // Rtp Byte 1 -> Byte 0 of HEVC NAL Header + // Rtp Byte 2 -> Byte 1 of HEVC NAL Header // Rtp Payload -> HEVC NAL bytes, so leave them unchanged // Set data position from byte 1 as byte 0 was ignored data.getData()[1] = (byte) nalHeader[0]; From aa6874655d63d684931d3957954931631934977e Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Fri, 11 Feb 2022 13:46:20 +0530 Subject: [PATCH 5/6] Add TODO for AggregationPacket mode --- .../androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java index 53c3a260d6b..28d5c7a33ab 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -188,6 +188,7 @@ private void processSingleNalUnitPacket(ParsableByteArray data) { */ @RequiresNonNull("trackOutput") private void processAggregationPacket(ParsableByteArray data) throws ParserException { + // TODO: Support AggregationPacket mode. throw ParserException.createForMalformedManifest( "need to implement processAggregationPacket", /* cause= */ null); From ca1c1c26d5e8828e2bcc726525c66c961928ecc0 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Tue, 15 Feb 2022 22:32:06 +0530 Subject: [PATCH 6/6] Fix some minor review comments in RtpH265Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 27 +++++++++---------- .../exoplayer/rtsp/reader/RtpH265Reader.java | 20 +++++--------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index afa3327efbe..5ca99fdeda4 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -44,10 +44,10 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; - private static final String PARAMETER_SPROP_H265_SPS = "sprop-sps"; - private static final String PARAMETER_SPROP_H265_PPS = "sprop-pps"; - private static final String PARAMETER_SPROP_H265_VPS = "sprop-vps"; - private static final String PARAMETER_SPROP_H265_MAX_DON_DIFF = "sprop-max-don-diff"; + private static final String PARAMETER_H265_SPROP_SPS = "sprop-sps"; + private static final String PARAMETER_H265_SPROP_PPS = "sprop-pps"; + private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps"; + private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -222,19 +222,19 @@ private static void processH264FmtpAttribute( private static void processH265FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { - if (fmtpAttributes.containsKey(PARAMETER_SPROP_H265_MAX_DON_DIFF)) { + if (fmtpAttributes.containsKey(PARAMETER_H265_SPROP_MAX_DON_DIFF)) { checkArgument( - Integer.parseInt(checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_MAX_DON_DIFF))) + Integer.parseInt(checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_MAX_DON_DIFF))) == 0, "non-zero sprop-max-don-diff is not supported"); } - checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); - String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); - checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_SPS)); - String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_SPS)); - checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_PPS)); - String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_PPS)); + checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_VPS)); + String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_VPS)); + checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_SPS)); + String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_SPS)); + checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_PPS)); + String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_PPS)); ImmutableList initializationData = ImmutableList.of( getInitializationDataFromParameterSet(spropVPS), @@ -248,8 +248,7 @@ private static void processH265FmtpAttribute( NalUnitUtil.parseH265SpsNalUnit( spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio); - formatBuilder.setHeight(spsData.height); - formatBuilder.setWidth(spsData.width); + formatBuilder.setHeight(spsData.height).setWidth(spsData.width); formatBuilder.setCodecs( CodecSpecificDataUtil.buildHevcCodecString( diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java index 28d5c7a33ab..f8159dc5295 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -68,10 +68,7 @@ /** Scratch for Fragmentation Unit RTP packets. */ private final ParsableByteArray fuScratchBuffer; - - private final ParsableByteArray nalStartCodeArray = - new ParsableByteArray(NalUnitUtil.NAL_START_CODE); - + private final ParsableByteArray nalStartCodeArray; private final RtpPayloadFormat payloadFormat; private @MonotonicNonNull TrackOutput trackOutput; @@ -84,8 +81,9 @@ /** Creates an instance. */ public RtpH265Reader(RtpPayloadFormat payloadFormat) { - this.payloadFormat = payloadFormat; fuScratchBuffer = new ParsableByteArray(); + nalStartCodeArray = new ParsableByteArray(NalUnitUtil.NAL_START_CODE); + this.payloadFormat = payloadFormat; firstReceivedTimestamp = C.TIME_UNSET; previousSequenceNumber = C.INDEX_UNSET; } @@ -235,11 +233,7 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header, RFC7798 Section 1.1.4 int fuHeader = data.getData()[2]; int nalUnitType = fuHeader & 0x3F; - byte nalHeader[] = new byte[2]; - nalHeader[0] = (byte) (nalUnitType << 1); // RFC7798 Section 1.1.4 - // layerId must be zero according to RFC7798 Section 1.1.4, so copying the tid only - nalHeader[1] = (byte) tid; boolean isFirstFuPacket = (fuHeader & 0x80) > 0; boolean isLastFuPacket = (fuHeader & 0x40) > 0; @@ -249,12 +243,12 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe // Overwrite a few bytes in Rtp buffer to get HEVC NAL Unit // Rtp Byte 0 -> Ignore - // Rtp Byte 1 -> Byte 0 of HEVC NAL Header - // Rtp Byte 2 -> Byte 1 of HEVC NAL Header + // Rtp Byte 1 -> nal_unit_type, RFC7798 Section 1.1.4 + // Rtp Byte 2 -> layerId required to be zero so copying only tid, RFC7798 Section 1.1.4 // Rtp Payload -> HEVC NAL bytes, so leave them unchanged // Set data position from byte 1 as byte 0 was ignored - data.getData()[1] = (byte) nalHeader[0]; - data.getData()[2] = (byte) nalHeader[1]; + data.getData()[1] = (byte) (nalUnitType << 1); + data.getData()[2] = (byte) tid; fuScratchBuffer.reset(data.getData()); fuScratchBuffer.setPosition(1); } else {