Remove late frame dropping in FrameProcessor

Currently, a frame is dropped if it's requested release time is in the past. This mode was added to support previewing. However, in normal ExoPlayer playback, slightly late frames (<30ms late) are also rendered. On MediaCodec side, this means calling `releaseOutputBuffer` with a release time in the past. PiperOrigin-RevId: 479615291
DiceTechnology · Oct 20, 2022 · 2188685 · 2188685
1 parent a6b9772
commit 2188685
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 34 deletions.
diff --git a/.../src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java b/.../src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java
@@ -158,7 +158,7 @@ public void controlledFrameRelease_withOneFrameRequestImmediateRelease_releasesF
   }
 
   @Test
-  public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception {
+  public void controlledFrameRelease_withLateFrame_releasesFrame() throws Exception {
     long originalPresentationTimeUs = 1234;
     long releaseTimeBeforeCurrentTimeNs = System.nanoTime() - 345678;
     AtomicLong actualPresentationTimeUs = new AtomicLong();
@@ -175,7 +175,9 @@ public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception {
 
     assertThat(frameProcessingException.get()).isNull();
     assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs);
-    assertThat(outputReleaseTimesNs).isEmpty();
+    assertThat(outputReleaseTimesNs).hasSize(1);
+    // The actual release time is determined by the FrameProcessor when releasing the frame.
+    assertThat(outputReleaseTimesNs.remove()).isAtLeast(releaseTimeBeforeCurrentTimeNs);
   }
 
   @Test

diff --git a/...aries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java b/...aries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java
@@ -150,10 +150,7 @@ public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
     frameProcessorListener.onOutputFrameAvailable(offsetPresentationTimeUs);
     if (releaseFramesAutomatically) {
       renderFrameToSurfaces(
-          inputTexture,
-          presentationTimeUs,
-          /* releaseTimeNs= */ offsetPresentationTimeUs * 1000,
-          /* dropLateFrame= */ false);
+          inputTexture, presentationTimeUs, /* releaseTimeNs= */ offsetPresentationTimeUs * 1000);
     } else {
       availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
     }
@@ -169,21 +166,11 @@ public void releaseOutputFrame(TextureInfo outputTexture) {
   @WorkerThread
   public void releaseOutputFrame(long releaseTimeNs) {
     checkState(!releaseFramesAutomatically);
-
-    boolean dropLateFrame = true;
-    if (releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY) {
-      dropLateFrame = false;
-      releaseTimeNs = System.nanoTime();
-    } else if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME) {
-      releaseTimeNs = C.TIME_UNSET;
-    }
-
     Pair<TextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
     renderFrameToSurfaces(
         /* inputTexture= */ oldestAvailableFrame.first,
         /* presentationTimeUs= */ oldestAvailableFrame.second,
-        releaseTimeNs,
-        dropLateFrame);
+        releaseTimeNs);
   }
 
   @Override
@@ -254,13 +241,9 @@ public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfac
   }
 
   private void renderFrameToSurfaces(
-      TextureInfo inputTexture,
-      long presentationTimeUs,
-      long releaseTimeNs,
-      boolean dropLateFrame) {
+      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
     try {
-      maybeRenderFrameToOutputSurface(
-          inputTexture, presentationTimeUs, releaseTimeNs, dropLateFrame);
+      maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
     } catch (FrameProcessingException | GlUtil.GlException e) {
       frameProcessorListener.onFrameProcessingError(
           FrameProcessingException.from(e, presentationTimeUs));
@@ -270,10 +253,11 @@ private void renderFrameToSurfaces(
   }
 
   private synchronized void maybeRenderFrameToOutputSurface(
-      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs, boolean dropLateFrame)
+      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
       throws FrameProcessingException, GlUtil.GlException {
-    if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
-      return; // Drop frames when there is no output surface.
+    if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME
+        || !ensureConfigured(inputTexture.width, inputTexture.height)) {
+      return; // Drop frames when requested, or there is no output surface.
     }
 
     EGLSurface outputEglSurface = this.outputEglSurface;
@@ -289,10 +273,12 @@ private synchronized void maybeRenderFrameToOutputSurface(
     GlUtil.clearOutputFrame();
     matrixTextureProcessor.drawFrame(inputTexture.texId, presentationTimeUs);
 
-    if (dropLateFrame && System.nanoTime() > releaseTimeNs) {
-      return;
-    }
-    EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, releaseTimeNs);
+    EGLExt.eglPresentationTimeANDROID(
+        eglDisplay,
+        outputEglSurface,
+        releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY
+            ? System.nanoTime()
+            : releaseTimeNs);
     EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
   }
 

diff --git a/library/common/src/main/java/androidx/media3/common/FrameProcessor.java b/library/common/src/main/java/androidx/media3/common/FrameProcessor.java
@@ -16,6 +16,7 @@
 package androidx.media3.common;
 
 import android.content.Context;
+import android.opengl.EGLExt;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import java.util.List;
@@ -172,10 +173,12 @@ interface Listener {
    * false} using the {@link Factory} and should be called exactly once for each frame that becomes
    * {@linkplain Listener#onOutputFrameAvailable(long) available}.
    *
-   * @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link
-   *     #DROP_OUTPUT_FRAME} to drop the frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to
-   *     release the frame immediately. If {@code releaseTimeNs} is after {@link System#nanoTime()}
-   *     at the time of the release, the frame is also dropped.
+   * <p>The {@code releaseTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID}
+   * depending on the implementation.
+   *
+   * @param releaseTimeNs The release time to use for the frame, in nanoseconds. The release time
+   *     can be before of after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the
+   *     frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to release the frame immediately.
    */
   void releaseOutputFrame(long releaseTimeNs);