Skip to content

Commit

Permalink
Display last frame when seeking to end of stream.
Browse files Browse the repository at this point in the history 
We currently don't display the last frame because the seek time is behind the
last frame's timestamps and it's thus marked as decodeOnly.

This case can be detected by checking whether all data sent to the codec is
marked as decodeOnly at the time we read the end of stream signal. If so, we
can re-enable the last frame. This should work for almost all cases because the
end-of-stream signal is read in the same feedInputBuffer loop as the last
frame and we therefore haven't released the last frame buffer yet.

Issue:#2568
PiperOrigin-RevId: 251425870
  • Loading branch information
@tonihei @ojw28
tonihei authored and ojw28 committed Jun 6, 2019
1 parent 44aa731 commit be88499
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 26 deletions.
2 changes: 2 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
and analytics reporting (TODO: link to developer guide page/blog post).
* Add basic DRM support to the Cast demo app.
* Offline: Add `Scheduler` implementation that uses `WorkManager`.
* Display last frame when seeking to end of stream
([#2568](https://github.com/google/ExoPlayer/issues/2568)).
* Assume that encrypted content requires secure decoders in renderer support
checks ([#5568](https://github.com/google/ExoPlayer/issues/5568)).
* Decoders: Prefer decoders that advertise format support over ones that do not,
Expand Down
5 changes: 3 additions & 2 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -691,7 +691,8 @@ protected boolean processOutputBuffer(
int bufferIndex,
int bufferFlags,
long bufferPresentationTimeUs,
boolean shouldSkip,
boolean isDecodeOnlyBuffer,
boolean isLastBuffer,
Format format)
throws ExoPlaybackException {
if (codecNeedsEosBufferTimestampWorkaround
Expand All @@ -707,7 +708,7 @@ protected boolean processOutputBuffer(
return true;
}

if (shouldSkip) {
if (isDecodeOnlyBuffer) {
codec.releaseOutputBuffer(bufferIndex, false);
decoderCounters.skippedOutputBufferCount++;
audioSink.handleDiscontinuity();
Expand Down
42 changes: 32 additions & 10 deletions library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,14 +328,16 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
private int inputIndex;
private int outputIndex;
private ByteBuffer outputBuffer;
private boolean shouldSkipOutputBuffer;
private boolean isDecodeOnlyOutputBuffer;
private boolean isLastOutputBuffer;
private boolean codecReconfigured;
@ReconfigurationState private int codecReconfigurationState;
@DrainState private int codecDrainState;
@DrainAction private int codecDrainAction;
private boolean codecReceivedBuffers;
private boolean codecReceivedEos;

private long lastBufferInStreamPresentationTimeUs;
private long largestQueuedPresentationTimeUs;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
private boolean waitingForKeys;
Expand Down Expand Up @@ -600,6 +602,8 @@ protected void releaseCodec() {
waitingForKeys = false;
codecHotswapDeadlineMs = C.TIME_UNSET;
decodeOnlyPresentationTimestamps.clear();
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
try {
if (codec != null) {
decoderCounters.decoderReleaseCount++;
Expand Down Expand Up @@ -706,10 +710,13 @@ protected boolean flushOrReleaseCodec() {
waitingForFirstSyncSample = true;
codecNeedsAdaptationWorkaroundBuffer = false;
shouldSkipAdaptationWorkaroundOutputBuffer = false;
shouldSkipOutputBuffer = false;
isDecodeOnlyOutputBuffer = false;
isLastOutputBuffer = false;

waitingForKeys = false;
decodeOnlyPresentationTimestamps.clear();
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
codecDrainState = DRAIN_STATE_NONE;
codecDrainAction = DRAIN_ACTION_NONE;
// Reconfiguration data sent shortly before the flush may not have been processed by the
Expand Down Expand Up @@ -883,7 +890,8 @@ private void initCodec(MediaCodecInfo codecInfo, MediaCrypto crypto) throws Exce
codecDrainAction = DRAIN_ACTION_NONE;
codecNeedsAdaptationWorkaroundBuffer = false;
shouldSkipAdaptationWorkaroundOutputBuffer = false;
shouldSkipOutputBuffer = false;
isDecodeOnlyOutputBuffer = false;
isLastOutputBuffer = false;
waitingForFirstSyncSample = true;

decoderCounters.decoderInitCount++;
Expand Down Expand Up @@ -1010,6 +1018,11 @@ private boolean feedInputBuffer() throws ExoPlaybackException {
result = readSource(formatHolder, buffer, false);
}

if (hasReadStreamToEnd()) {
// Notify output queue of the last buffer's timestamp.
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
}

if (result == C.RESULT_NOTHING_READ) {
return false;
}
Expand Down Expand Up @@ -1082,6 +1095,8 @@ private boolean feedInputBuffer() throws ExoPlaybackException {
formatQueue.add(presentationTimeUs, inputFormat);
waitingForFirstSampleInFormat = false;
}
largestQueuedPresentationTimeUs =
Math.max(largestQueuedPresentationTimeUs, presentationTimeUs);

buffer.flip();
onQueueInputBuffer(buffer);
Expand Down Expand Up @@ -1456,7 +1471,9 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
outputBuffer.position(outputBufferInfo.offset);
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
}
shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs);
isDecodeOnlyOutputBuffer = isDecodeOnlyBuffer(outputBufferInfo.presentationTimeUs);
isLastOutputBuffer =
lastBufferInStreamPresentationTimeUs == outputBufferInfo.presentationTimeUs;
updateOutputFormatForTime(outputBufferInfo.presentationTimeUs);
}

Expand All @@ -1472,7 +1489,8 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
outputIndex,
outputBufferInfo.flags,
outputBufferInfo.presentationTimeUs,
shouldSkipOutputBuffer,
isDecodeOnlyOutputBuffer,
isLastOutputBuffer,
outputFormat);
} catch (IllegalStateException e) {
processEndOfStream();
Expand All @@ -1492,7 +1510,8 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
outputIndex,
outputBufferInfo.flags,
outputBufferInfo.presentationTimeUs,
shouldSkipOutputBuffer,
isDecodeOnlyOutputBuffer,
isLastOutputBuffer,
outputFormat);
}

Expand Down Expand Up @@ -1559,7 +1578,9 @@ private void processOutputBuffersChanged() {
* @param bufferIndex The index of the output buffer.
* @param bufferFlags The flags attached to the output buffer.
* @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds.
* @param shouldSkip Whether the buffer should be skipped (i.e. not rendered).
* @param isDecodeOnlyBuffer Whether the buffer was marked with {@link C#BUFFER_FLAG_DECODE_ONLY}
* by the source.
* @param isLastBuffer Whether the buffer is the last sample of the current stream.
* @param format The format associated with the buffer.
* @return Whether the output buffer was fully processed (e.g. rendered or skipped).
* @throws ExoPlaybackException If an error occurs processing the output buffer.
Expand All @@ -1572,7 +1593,8 @@ protected abstract boolean processOutputBuffer(
int bufferIndex,
int bufferFlags,
long bufferPresentationTimeUs,
boolean shouldSkip,
boolean isDecodeOnlyBuffer,
boolean isLastBuffer,
Format format)
throws ExoPlaybackException;

Expand Down Expand Up @@ -1652,7 +1674,7 @@ private void updateDrmSessionOrReinitializeCodecV23() throws ExoPlaybackExceptio
codecDrainAction = DRAIN_ACTION_NONE;
}

private boolean shouldSkipOutputBuffer(long presentationTimeUs) {
private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
// We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
// box presentationTimeUs, creating a Long object that would need to be garbage collected.
int size = decodeOnlyPresentationTimestamps.size();
Expand Down
2 changes: 1 addition & 1 deletion library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ private void startLoading() {
if (prepared) {
SeekMap seekMap = getPreparedState().seekMap;
Assertions.checkState(isPendingReset());
if (durationUs != C.TIME_UNSET && pendingResetPositionUs >= durationUs) {
if (durationUs != C.TIME_UNSET && pendingResetPositionUs > durationUs) {
loadingFinished = true;
pendingResetPositionUs = C.TIME_UNSET;
return;
Expand Down
25 changes: 15 additions & 10 deletions library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,8 @@ protected boolean processOutputBuffer(
int bufferIndex,
int bufferFlags,
long bufferPresentationTimeUs,
boolean shouldSkip,
boolean isDecodeOnlyBuffer,
boolean isLastBuffer,
Format format)
throws ExoPlaybackException {
if (initialPositionUs == C.TIME_UNSET) {
Expand All @@ -721,7 +722,7 @@ protected boolean processOutputBuffer(

long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;

if (shouldSkip) {
if (isDecodeOnlyBuffer && !isLastBuffer) {
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
return true;
}
Expand Down Expand Up @@ -769,10 +770,10 @@ && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeU
bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs);
earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;

if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs)
if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer)
&& maybeDropBuffersToKeyframe(codec, bufferIndex, presentationTimeUs, positionUs)) {
return false;
} else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) {
} else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) {
dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
return true;
}
Expand Down Expand Up @@ -840,8 +841,8 @@ private void notifyFrameMetadataListener(

/**
* Returns the offset that should be subtracted from {@code bufferPresentationTimeUs} in {@link
* #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, Format)} to
* get the playback position with respect to the media.
* #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, boolean,
* Format)} to get the playback position with respect to the media.
*/
protected long getOutputStreamOffsetUs() {
return outputStreamOffsetUs;
Expand Down Expand Up @@ -893,9 +894,11 @@ protected void onProcessedOutputBuffer(long presentationTimeUs) {
* indicates that the buffer is late.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* measured at the start of the current iteration of the rendering loop.
* @param isLastBuffer Whether the buffer is the last buffer in the current stream.
*/
protected boolean shouldDropOutputBuffer(long earlyUs, long elapsedRealtimeUs) {
return isBufferLate(earlyUs);
protected boolean shouldDropOutputBuffer(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
return isBufferLate(earlyUs) && !isLastBuffer;
}

/**
Expand All @@ -906,9 +909,11 @@ protected boolean shouldDropOutputBuffer(long earlyUs, long elapsedRealtimeUs) {
* negative value indicates that the buffer is late.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* measured at the start of the current iteration of the rendering loop.
* @param isLastBuffer Whether the buffer is the last buffer in the current stream.
*/
protected boolean shouldDropBuffersToKeyframe(long earlyUs, long elapsedRealtimeUs) {
return isBufferVeryLate(earlyUs);
protected boolean shouldDropBuffersToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
return isBufferVeryLate(earlyUs) && !isLastBuffer;
}

/**
Expand Down
8 changes: 5 additions & 3 deletions testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,15 @@ protected boolean processOutputBuffer(
int bufferIndex,
int bufferFlags,
long bufferPresentationTimeUs,
boolean shouldSkip,
boolean isDecodeOnlyBuffer,
boolean isLastBuffer,
Format format)
throws ExoPlaybackException {
if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) {
// After the codec has been initialized, don't render the first frame until we've caught up
// to the playback position. Else test runs on devices that do not support dummy surface
// will drop frames between rendering the first one and catching up [Internal: b/66494991].
shouldSkip = true;
isDecodeOnlyBuffer = true;
}
return super.processOutputBuffer(
positionUs,
Expand All @@ -183,7 +184,8 @@ protected boolean processOutputBuffer(
bufferIndex,
bufferFlags,
bufferPresentationTimeUs,
shouldSkip,
isDecodeOnlyBuffer,
isLastBuffer,
format);
}

Expand Down

0 comments on commit be88499

Please sign in to comment.