diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index 97bfded9797..2964ecdcf0e 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -66,11 +66,20 @@ public void imageInput_queueThreeBitmaps_outputsCorrectNumberOfFrames() throws E videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2); + readBitmap(ORIGINAL_PNG_ASSET_PATH), + C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 2); videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(SCALE_WIDE_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3); + readBitmap(SCALE_WIDE_PNG_ASSET_PATH), + 2 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 3); videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4); + readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), + 3 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 4); videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); int actualFrameCount = framesProduced.get(); @@ -87,6 +96,7 @@ public void imageInput_queueTwentyBitmaps_outputsCorrectNumberOfFrames() throws videoFrameProcessorTestRunner.queueInputBitmap( readBitmap(ORIGINAL_PNG_ASSET_PATH), /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, /* frameRate= */ 1); } videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); @@ -95,6 +105,63 @@ public void imageInput_queueTwentyBitmaps_outputsCorrectNumberOfFrames() throws assertThat(actualFrameCount).isEqualTo(/* expected= */ 20); } + @RequiresNonNull("framesProduced") + @Test + public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + String testId = + "imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs"; + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add) + .build(); + + long offsetUs = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); + assertThat(actualPresentationTimesUs) + .containsExactly(offsetUs, offsetUs + C.MICROS_PER_SECOND / 2) + .inOrder(); + } + + @RequiresNonNull("framesProduced") + @Test + public void imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + String testId = "imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs"; + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add) + .build(); + + long offsetUs1 = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs1, + /* frameRate= */ 2); + long offsetUs2 = 2_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(SCALE_WIDE_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs2, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); + assertThat(actualPresentationTimesUs) + .containsExactly( + offsetUs1, + offsetUs1 + C.MICROS_PER_SECOND / 2, + offsetUs2, + offsetUs2 + C.MICROS_PER_SECOND / 2) + .inOrder(); + } + @RequiresNonNull("framesProduced") @Test public void @@ -111,11 +178,13 @@ public void imageInput_queueTwentyBitmaps_outputsCorrectNumberOfFrames() throws videoFrameProcessorTestRunner.queueInputBitmap( readBitmap(ORIGINAL_PNG_ASSET_PATH), /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, /* frameRate= */ 2); videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); videoFrameProcessorTestRunner.queueInputBitmap( readBitmap(ORIGINAL_PNG_ASSET_PATH), /* durationUs= */ 2 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, /* frameRate= */ 3); assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java index d69712f0e28..8c5cca597e9 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java @@ -120,7 +120,7 @@ public void noEffects_withImageInput_matchesGoldenFile() throws Exception { Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH); videoFrameProcessorTestRunner.queueInputBitmap( - originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); + originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1); Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -148,7 +148,7 @@ public void wrappedCrop_withImageInput_matchesGoldenFile() throws Exception { Bitmap expectedBitmap = readBitmap(IMAGE_TO_CROPPED_VIDEO_PNG_ASSET_PATH); videoFrameProcessorTestRunner.queueInputBitmap( - originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); + originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1); Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 950971b3c22..f3898e74589 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -79,9 +79,9 @@ public void onReadyToAcceptInputFrame() { @Override public void queueInputBitmap( - Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { + Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) { videoFrameProcessingTaskExecutor.submit( - () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr)); + () -> setupBitmap(inputBitmap, durationUs, offsetUs, frameRate, useHdr)); } @Override @@ -116,7 +116,8 @@ public void release() { // Methods that must be called on the GL thread. - private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr) + private void setupBitmap( + Bitmap bitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) throws VideoFrameProcessingException { this.useHdr = useHdr; if (inputEnded) { @@ -124,7 +125,7 @@ private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolea } int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); double frameDurationUs = C.MICROS_PER_SECOND / frameRate; - pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, frameDurationUs, framesToAdd)); + pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, offsetUs, frameDurationUs, framesToAdd)); maybeQueueToShaderProgram(); } @@ -138,6 +139,7 @@ private void maybeQueueToShaderProgram() throws VideoFrameProcessingException { if (framesToQueueForCurrentBitmap == 0) { Bitmap bitmap = currentBitmapInfo.bitmap; framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames; + currentPresentationTimeUs = currentBitmapInfo.offsetUs; int currentTexId; try { if (currentGlTextureInfo != null) { @@ -189,11 +191,14 @@ private void maybeSignalEndOfOutput() { /** Information to generate all the frames associated with a specific {@link Bitmap}. */ private static final class BitmapFrameSequenceInfo { public final Bitmap bitmap; + public final long offsetUs; public final double frameDurationUs; public final int numberOfFrames; - public BitmapFrameSequenceInfo(Bitmap bitmap, double frameDurationUs, int numberOfFrames) { + public BitmapFrameSequenceInfo( + Bitmap bitmap, long offsetUs, double frameDurationUs, int numberOfFrames) { this.bitmap = bitmap; + this.offsetUs = offsetUs; this.frameDurationUs = frameDurationUs; this.numberOfFrames = numberOfFrames; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 5dbec3d81b5..77463e4e644 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -17,6 +17,7 @@ import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE; import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; @@ -253,6 +254,7 @@ public DefaultVideoFrameProcessor create( private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile boolean inputStreamEnded; + private volatile boolean hasRefreshedNextInputFrameInfo; private DefaultVideoFrameProcessor( EGLDisplay eglDisplay, @@ -321,7 +323,16 @@ public void setInputDefaultBufferSize(int width, int height) { @Override public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { - inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false); + checkState( + hasRefreshedNextInputFrameInfo, + "setInputFrameInfo must be called before queueing another bitmap"); + inputHandler.queueInputBitmap( + inputBitmap, + durationUs, + checkNotNull(nextInputFrameInfo).offsetToAddUs, + frameRate, + /* useHdr= */ false); + hasRefreshedNextInputFrameInfo = false; } @Override @@ -332,6 +343,7 @@ public Surface getInputSurface() { @Override public void setInputFrameInfo(FrameInfo inputFrameInfo) { nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); + hasRefreshedNextInputFrameInfo = true; } @Override @@ -341,6 +353,7 @@ public void registerInputFrame() { nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); inputHandler.registerInputFrame(nextInputFrameInfo); + hasRefreshedNextInputFrameInfo = false; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 0750fb7ca2f..b84cc8d75dd 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -17,6 +17,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; +import android.graphics.Bitmap; import android.graphics.SurfaceTexture; import android.view.Surface; import androidx.annotation.Nullable; @@ -109,6 +110,12 @@ public void setDefaultBufferSize(int width, int height) { surfaceTexture.setDefaultBufferSize(width, height); } + @Override + public void queueInputBitmap( + Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) { + throw new UnsupportedOperationException(); + } + @Override public Surface getInputSurface() { return surface; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java b/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java index 4d3b5f5bd1e..c9e689d543a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java @@ -39,10 +39,16 @@ default void setDefaultBufferSize(int width, int height) { /** * Provides an input {@link Bitmap} to put into the video frames. * - * @see VideoFrameProcessor#queueInputBitmap + * @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}. + * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds. + * @param offsetUs The offset, from the start of the input stream, to apply for the {@code + * inputBitmap} in microseconds. + * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per + * second. + * @param useHdr Whether input and/or output colors are HDR. */ default void queueInputBitmap( - Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { + Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) { throw new UnsupportedOperationException(); } diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 7ca8c84e921..6549e8d7470 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -333,10 +333,12 @@ public void onFrameDecoded(MediaFormat mediaFormat) { return endFrameProcessingAndGetImage(); } - public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { + public void queueInputBitmap( + Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) { videoFrameProcessor.setInputFrameInfo( new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) .setPixelWidthHeightRatio(pixelWidthHeightRatio) + .setOffsetToAddUs(offsetToAddUs) .build()); videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); }