From c4ce6b7511990e21b4e37a2099d4f5df3e309739 Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 8 Aug 2022 08:01:44 +0000 Subject: [PATCH] Merge pull request #119 from ittiam-systems:rtp_h263_test_and_fix PiperOrigin-RevId: 463146426 (cherry picked from commit e54d2f56584d2f62d9bc26edf057408a9be84228) --- RELEASENOTES.md | 3 + .../exoplayer/rtsp/reader/RtpH263Reader.java | 66 ++++-- .../rtsp/reader/RtpH263ReaderTest.java | 222 ++++++++++++++++++ 3 files changed, 274 insertions(+), 17 deletions(-) create mode 100644 libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a2abb2739fa..271c68aa7e8 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -30,6 +30,9 @@ small icon ([#104](https://github.com/androidx/media/issues/104)). * Ensure commands sent before `MediaController.release()` are not dropped ([#99](https://github.com/androidx/media/issues/99)). +* RTSP: + * Add H263 fragmented packet handling + ([#119](https://github.com/androidx/media/pull/119)). ### 1.0.0-beta02 (2022-07-15) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH263Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH263Reader.java index 4aedc65aad8..1c62d7ab5e2 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH263Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH263Reader.java @@ -15,6 +15,8 @@ */ package androidx.media3.exoplayer.rtsp.reader; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import androidx.media3.common.C; @@ -61,6 +63,12 @@ private boolean isKeyFrame; private boolean isOutputFormatSet; private long startTimeOffsetUs; + private long fragmentedSampleTimeUs; + /** + * Whether the first packet of a H263 frame is received, it mark the start of a H263 partition. A + * H263 frame can be split into multiple RTP packets. + */ + private boolean gotFirstPacketOfH263Frame; /** Creates an instance. */ public RtpH263Reader(RtpPayloadFormat payloadFormat) { @@ -76,7 +84,10 @@ public void createTracks(ExtractorOutput extractorOutput, int trackId) { } @Override - public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + checkState(firstReceivedTimestamp == C.TIME_UNSET); + firstReceivedTimestamp = timestamp; + } @Override public void consume( @@ -103,6 +114,12 @@ public void consume( } if (pBitIsSet) { + if (gotFirstPacketOfH263Frame && fragmentedSampleSizeBytes > 0) { + // Received new H263 fragment, output data of previous fragment to decoder. + outputSampleMetadataForFragmentedPackets(); + } + gotFirstPacketOfH263Frame = true; + int payloadStartCode = data.peekUnsignedByte() & 0xFC; // Packets that begin with a Picture Start Code(100000). Refer RFC4629 Section 6.1. if (payloadStartCode < PICTURE_START_CODE) { @@ -113,10 +130,10 @@ public void consume( data.getData()[currentPosition] = 0; data.getData()[currentPosition + 1] = 0; data.setPosition(currentPosition); - } else { + } else if (gotFirstPacketOfH263Frame) { // Check that this packet is in the sequence of the previous packet. int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); - if (sequenceNumber != expectedSequenceNumber) { + if (sequenceNumber < expectedSequenceNumber) { Log.w( TAG, Util.formatInvariant( @@ -125,6 +142,12 @@ public void consume( expectedSequenceNumber, sequenceNumber)); return; } + } else { + Log.w( + TAG, + "First payload octet of the H263 packet is not the beginning of a new H263 partition," + + " Dropping current packet."); + return; } if (fragmentedSampleSizeBytes == 0) { @@ -141,20 +164,10 @@ public void consume( // Write the video sample. trackOutput.sampleData(data, fragmentSize); fragmentedSampleSizeBytes += fragmentSize; + fragmentedSampleTimeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); if (rtpMarker) { - if (firstReceivedTimestamp == C.TIME_UNSET) { - firstReceivedTimestamp = timestamp; - } - long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); - trackOutput.sampleMetadata( - timeUs, - isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0, - fragmentedSampleSizeBytes, - /* offset= */ 0, - /* cryptoData= */ null); - fragmentedSampleSizeBytes = 0; - isKeyFrame = false; + outputSampleMetadataForFragmentedPackets(); } previousSequenceNumber = sequenceNumber; } @@ -167,8 +180,8 @@ public void seek(long nextRtpTimestamp, long timeUs) { } /** - * Parses and set VOP Coding type and resolution. The {@link ParsableByteArray#position} is - * preserved. + * Parses and set VOP Coding type and resolution. The {@linkplain ParsableByteArray#getPosition() + * position} is preserved. */ private void parseVopHeader(ParsableByteArray data, boolean gotResolution) { // Picture Segment Packets (RFC4629 Section 6.1). @@ -211,6 +224,25 @@ private void parseVopHeader(ParsableByteArray data, boolean gotResolution) { isKeyFrame = false; } + /** + * Outputs sample metadata of the received fragmented packets. + * + *

Call this method only after receiving an end of a H263 partition. + */ + private void outputSampleMetadataForFragmentedPackets() { + checkNotNull(trackOutput) + .sampleMetadata( + fragmentedSampleTimeUs, + isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* cryptoData= */ null); + fragmentedSampleSizeBytes = 0; + fragmentedSampleTimeUs = C.TIME_UNSET; + isKeyFrame = false; + gotFirstPacketOfH263Frame = false; + } + private static long toSampleUs( long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { return startTimeOffsetUs diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java new file mode 100644 index 00000000000..4c1f4efde06 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Util.getBytesFromHexString; +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.test.utils.FakeExtractorOutput; +import androidx.media3.test.utils.FakeTrackOutput; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Bytes; +import java.util.Arrays; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link RtpH263Reader}. */ +@RunWith(AndroidJUnit4.class) +public final class RtpH263ReaderTest { + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + private static final byte[] FRAME_1_FRAGMENT_1_DATA = + getBytesFromHexString("80020c0419b7b7d9591f03023e0c37b"); + private static final long PARTITION_1_RTP_TIMESTAMP = 2599168056L; + private static final RtpPacket PACKET_FRAME_1_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_1_RTP_TIMESTAMP) + .setSequenceNumber(40289) + .setMarker(false) + .setPayloadData( + Bytes.concat( + /*payload header */ getBytesFromHexString("0400"), FRAME_1_FRAGMENT_1_DATA)) + .build(); + private static final byte[] FRAME_1_FRAGMENT_2_DATA = + getBytesFromHexString("03140e0e77d5e83021a0c37"); + private static final RtpPacket PACKET_FRAME_1_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_1_RTP_TIMESTAMP) + .setSequenceNumber(40290) + .setMarker(true) + .setPayloadData( + Bytes.concat( + /*payload header */ getBytesFromHexString("0000"), FRAME_1_FRAGMENT_2_DATA)) + .build(); + // Needs to add 0000 to byte stream, refer to RFC4629 Section 6.1.1. + private static final byte[] FRAME_1_DATA = + Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA, FRAME_1_FRAGMENT_2_DATA); + + private static final byte[] FRAME_2_FRAGMENT_1_DATA = + getBytesFromHexString("800a0e023ffffffffffffffffff"); + private static final long PARTITION_2_RTP_TIMESTAMP = 2599168344L; + private static final RtpPacket PACKET_FRAME_2_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_2_RTP_TIMESTAMP) + .setSequenceNumber(40291) + .setMarker(false) + .setPayloadData( + Bytes.concat( + /*payload header */ getBytesFromHexString("0400"), FRAME_2_FRAGMENT_1_DATA)) + .build(); + private static final byte[] FRAME_2_FRAGMENT_2_DATA = + getBytesFromHexString("830df80c501839dfccdbdbecac"); + private static final RtpPacket PACKET_FRAME_2_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_2_RTP_TIMESTAMP) + .setSequenceNumber(40292) + .setMarker(true) + .setPayloadData( + Bytes.concat( + /*payload header */ getBytesFromHexString("0000"), FRAME_2_FRAGMENT_2_DATA)) + .build(); + private static final byte[] FRAME_2_DATA = + Bytes.concat(getBytesFromHexString("0000"), FRAME_2_FRAGMENT_1_DATA, FRAME_2_FRAGMENT_2_DATA); + + private static final long PARTITION_2_PRESENTATION_TIMESTAMP_US = + Util.scaleLargeTimestamp( + (PARTITION_2_RTP_TIMESTAMP - PARTITION_1_RTP_TIMESTAMP), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + + private static final RtpPayloadFormat H263_FORMAT = + new RtpPayloadFormat( + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H263) + .setWidth(352) + .setHeight(288) + .build(), + /* rtpPayloadType= */ 96, + /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY, + /* fmtpParameters= */ ImmutableMap.of()); + + private FakeExtractorOutput extractorOutput; + + @Before + public void setUp() { + extractorOutput = new FakeExtractorOutput(); + } + + @Test + public void consume_validPackets() { + RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT); + h263Reader.createTracks(extractorOutput, /* trackId= */ 0); + h263Reader.onReceivingFirstPacket( + PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_DATA); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_fragmentedFrameMissingFirstFragment() { + RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT); + h263Reader.createTracks(extractorOutput, /* trackId= */ 0); + h263Reader.onReceivingFirstPacket( + PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(1); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_fragmentedFrameMissingBoundaryFragment() { + RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT); + h263Reader.createTracks(extractorOutput, /* trackId= */ 0); + h263Reader.onReceivingFirstPacket( + PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)) + .isEqualTo(Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA)); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_outOfOrderPackets() { + RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT); + h263Reader.createTracks(extractorOutput, /* trackId= */ 0); + h263Reader.onReceivingFirstPacket( + PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1); + consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2); + consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)) + .isEqualTo(Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA)); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + private static void consume(RtpH263Reader h263Reader, RtpPacket rtpPacket) { + rtpPacket = copyPacket(rtpPacket); + h263Reader.consume( + new ParsableByteArray(rtpPacket.payloadData), + rtpPacket.timestamp, + rtpPacket.sequenceNumber, + rtpPacket.marker); + } + + private static RtpPacket copyPacket(RtpPacket packet) { + RtpPacket.Builder builder = + new RtpPacket.Builder() + .setPadding(packet.padding) + .setMarker(packet.marker) + .setPayloadType(packet.payloadType) + .setSequenceNumber(packet.sequenceNumber) + .setTimestamp(packet.timestamp) + .setSsrc(packet.ssrc); + + if (packet.csrc.length > 0) { + builder.setCsrc(Arrays.copyOf(packet.csrc, packet.csrc.length)); + } + if (packet.payloadData.length > 0) { + builder.setPayloadData(Arrays.copyOf(packet.payloadData, packet.payloadData.length)); + } + return builder.build(); + } +}