Skip to content

Commit

Permalink
Use CRYPTO_ASYNC mode for video from API 34
Browse files Browse the repository at this point in the history 
This allows us to remove the additional thread we create
for asynchronous buffer queuing and use a synchronized
queueing approach again.

This API looks strictly beneficial on all tested devices,
but since this code path in MediaCodec has not been used
widely, we leave an opt-out flag for now.

PiperOrigin-RevId: 591867472
  • Loading branch information
@tonihei @copybara-github
tonihei authored and copybara-github committed Dec 18, 2023
1 parent bb7aa2f commit e5aa692
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 38 deletions.
14 changes: 14 additions & 0 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,20 @@ public DefaultRenderersFactory forceDisableMediaCodecAsynchronousQueueing() {
return this;
}

/**
* Sets whether to enable {@link MediaCodec#CONFIGURE_FLAG_USE_CRYPTO_ASYNC} on API 34 and above
* when operating the codec in asynchronous mode.
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*/
@CanIgnoreReturnValue
public DefaultRenderersFactory experimentalSetMediaCodecAsyncCryptoFlagEnabled(
boolean enableAsyncCryptoFlag) {
codecAdapterFactory.experimentalSetAsyncCryptoFlagEnabled(enableAsyncCryptoFlag);
return this;
}

/**
* Sets whether to enable fallback to lower-priority decoders if decoder initialization fails.
* This may result in using a decoder that is less efficient or slower than the primary decoder.
Expand Down
52 changes: 43 additions & 9 deletions ...yer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,16 @@
import android.os.HandlerThread;
import android.os.PersistableBundle;
import android.view.Surface;
import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.CryptoInfo;
import com.google.common.base.Supplier;
import java.io.IOException;
Expand All @@ -54,6 +58,8 @@ public static final class Factory implements MediaCodecAdapter.Factory {
private final Supplier<HandlerThread> callbackThreadSupplier;
private final Supplier<HandlerThread> queueingThreadSupplier;

private boolean enableSynchronousBufferQueueingWithAsyncCryptoFlag;

/**
* Creates an factory for {@link AsynchronousMediaCodecAdapter} instances.
*
Expand All @@ -74,6 +80,18 @@ public Factory(@C.TrackType int trackType) {
Supplier<HandlerThread> queueingThreadSupplier) {
this.callbackThreadSupplier = callbackThreadSupplier;
this.queueingThreadSupplier = queueingThreadSupplier;
enableSynchronousBufferQueueingWithAsyncCryptoFlag = true;
}

/**
* Sets whether to enable {@link MediaCodec#CONFIGURE_FLAG_USE_CRYPTO_ASYNC} on API 34 and
* above.
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*/
public void experimentalSetAsyncCryptoFlagEnabled(boolean enableAsyncCryptoFlag) {
enableSynchronousBufferQueueingWithAsyncCryptoFlag = enableAsyncCryptoFlag;
}

@Override
Expand All @@ -85,15 +103,21 @@ public AsynchronousMediaCodecAdapter createAdapter(Configuration configuration)
try {
TraceUtil.beginSection("createCodec:" + codecName);
codec = MediaCodec.createByCodecName(codecName);
int flags = configuration.flags;
MediaCodecBufferEnqueuer bufferEnqueuer;
if (enableSynchronousBufferQueueingWithAsyncCryptoFlag
&& useSynchronousBufferQueueingWithAsyncCryptoFlag(configuration.format)) {
bufferEnqueuer = new SynchronousMediaCodecBufferEnqueuer(codec);
flags |= MediaCodec.CONFIGURE_FLAG_USE_CRYPTO_ASYNC;
} else {
bufferEnqueuer =
new AsynchronousMediaCodecBufferEnqueuer(codec, queueingThreadSupplier.get());
}
codecAdapter =
new AsynchronousMediaCodecAdapter(
codec, callbackThreadSupplier.get(), queueingThreadSupplier.get());
new AsynchronousMediaCodecAdapter(codec, callbackThreadSupplier.get(), bufferEnqueuer);
TraceUtil.endSection();
codecAdapter.initialize(
configuration.mediaFormat,
configuration.surface,
configuration.crypto,
configuration.flags);
configuration.mediaFormat, configuration.surface, configuration.crypto, flags);
return codecAdapter;
} catch (Exception e) {
if (codecAdapter != null) {
Expand All @@ -104,6 +128,16 @@ public AsynchronousMediaCodecAdapter createAdapter(Configuration configuration)
throw e;
}
}

@ChecksSdkIntAtLeast(api = 34)
private static boolean useSynchronousBufferQueueingWithAsyncCryptoFlag(Format format) {
if (Util.SDK_INT < 34) {
return false;
}
// TODO: b/316565675 - Remove restriction to video once MediaCodec supports
// CONFIGURE_FLAG_USE_CRYPTO_ASYNC for audio too
return MimeTypes.isVideo(format.sampleMimeType);
}
}

@Documented
Expand All @@ -118,15 +152,15 @@ public AsynchronousMediaCodecAdapter createAdapter(Configuration configuration)

private final MediaCodec codec;
private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback;
private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer;
private final MediaCodecBufferEnqueuer bufferEnqueuer;
private boolean codecReleased;
private @State int state;

private AsynchronousMediaCodecAdapter(
MediaCodec codec, HandlerThread callbackThread, HandlerThread enqueueingThread) {
MediaCodec codec, HandlerThread callbackThread, MediaCodecBufferEnqueuer bufferEnqueuer) {
this.codec = codec;
this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread);
this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread);
this.bufferEnqueuer = bufferEnqueuer;
this.state = STATE_CREATED;
}

Expand Down
40 changes: 11 additions & 29 deletions .../main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package androidx.media3.exoplayer.mediacodec;

import static androidx.annotation.VisibleForTesting.NONE;
Expand All @@ -39,13 +38,11 @@
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
* Performs {@link MediaCodec} input buffer queueing on a background thread.
*
* <p>The implementation of this class assumes that its public methods will be called from the same
* thread.
* Performs {@link MediaCodec} input buffer queueing on a background thread. This is required on API
* 33 and below because queuing secure buffers blocks until decryption is complete.
*/
@RequiresApi(23)
class AsynchronousMediaCodecBufferEnqueuer {
/* package */ class AsynchronousMediaCodecBufferEnqueuer implements MediaCodecBufferEnqueuer {

private static final int MSG_QUEUE_INPUT_BUFFER = 0;
private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1;
Expand Down Expand Up @@ -83,11 +80,7 @@ public AsynchronousMediaCodecBufferEnqueuer(MediaCodec codec, HandlerThread queu
pendingRuntimeException = new AtomicReference<>();
}

/**
* Starts this instance.
*
* <p>Call this method after creating an instance and before queueing input buffers.
*/
@Override
public void start() {
if (!started) {
handlerThread.start();
Expand All @@ -102,11 +95,7 @@ public void handleMessage(Message msg) {
}
}

/**
* Submits an input buffer for decoding.
*
* @see android.media.MediaCodec#queueInputBuffer
*/
@Override
public void queueInputBuffer(
int index, int offset, int size, long presentationTimeUs, int flags) {
maybeThrowException();
Expand All @@ -116,15 +105,7 @@ public void queueInputBuffer(
message.sendToTarget();
}

/**
* Submits an input buffer that potentially contains encrypted data for decoding.
*
* <p>Note: This method behaves as {@link MediaCodec#queueSecureInputBuffer} with the difference
* that {@code info} is of type {@link CryptoInfo} and not {@link
* android.media.MediaCodec.CryptoInfo}.
*
* @see android.media.MediaCodec#queueSecureInputBuffer
*/
@Override
public void queueSecureInputBuffer(
int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
maybeThrowException();
Expand All @@ -136,12 +117,13 @@ public void queueSecureInputBuffer(
message.sendToTarget();
}

@Override
public void setParameters(Bundle params) {
maybeThrowException();
castNonNull(handler).obtainMessage(MSG_SET_PARAMETERS, params).sendToTarget();
}

/** Flushes the instance. */
@Override
public void flush() {
if (started) {
try {
Expand All @@ -155,7 +137,7 @@ public void flush() {
}
}

/** Shuts down the instance. Make sure to call this method to release its internal resources. */
@Override
public void shutdown() {
if (started) {
flush();
Expand All @@ -164,12 +146,12 @@ public void shutdown() {
started = false;
}

/** Blocks the current thread until all input buffers pending queueing are submitted. */
@Override
public void waitUntilQueueingComplete() throws InterruptedException {
blockUntilHandlerThreadIsIdle();
}

/** Throw any exception that occurred on the enqueuer's background queueing thread. */
@Override
public void maybeThrowException() {
@Nullable RuntimeException exception = pendingRuntimeException.getAndSet(null);
if (exception != null) {
Expand Down
21 changes: 21 additions & 0 deletions ...er/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@
@Nullable
private MediaCodec.CodecException mediaCodecException;

@GuardedBy("lock")
@Nullable
private MediaCodec.CryptoException mediaCodecCryptoException;

@GuardedBy("lock")
private long pendingFlushCount;

Expand Down Expand Up @@ -228,6 +232,13 @@ public void onError(MediaCodec codec, MediaCodec.CodecException e) {
}
}

@Override
public void onCryptoError(MediaCodec codec, MediaCodec.CryptoException e) {
synchronized (lock) {
mediaCodecCryptoException = e;
}
}

@Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
synchronized (lock) {
Expand Down Expand Up @@ -287,6 +298,7 @@ private void addOutputFormat(MediaFormat mediaFormat) {
private void maybeThrowException() {
maybeThrowInternalException();
maybeThrowMediaCodecException();
maybeThrowMediaCodecCryptoException();
}

@GuardedBy("lock")
Expand All @@ -307,6 +319,15 @@ private void maybeThrowMediaCodecException() {
}
}

@GuardedBy("lock")
private void maybeThrowMediaCodecCryptoException() {
if (mediaCodecCryptoException != null) {
MediaCodec.CryptoException cryptoException = mediaCodecCryptoException;
mediaCodecCryptoException = null;
throw cryptoException;
}
}

private void setInternalException(IllegalStateException e) {
synchronized (lock) {
internalException = e;
Expand Down
18 changes: 18 additions & 0 deletions ...r/src/main/java/androidx/media3/exoplayer/mediacodec/DefaultMediaCodecAdapterFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import static java.lang.annotation.ElementType.TYPE_USE;

import android.media.MediaCodec;
import androidx.annotation.IntDef;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Log;
Expand Down Expand Up @@ -54,9 +55,11 @@ public final class DefaultMediaCodecAdapterFactory implements MediaCodecAdapter.
private static final String TAG = "DMCodecAdapterFactory";

private @Mode int asynchronousMode;
private boolean asyncCryptoFlagEnabled;

public DefaultMediaCodecAdapterFactory() {
asynchronousMode = MODE_DEFAULT;
asyncCryptoFlagEnabled = true;
}

/**
Expand All @@ -83,6 +86,20 @@ public DefaultMediaCodecAdapterFactory forceDisableAsynchronous() {
return this;
}

/**
* Sets whether to enable {@link MediaCodec#CONFIGURE_FLAG_USE_CRYPTO_ASYNC} on API 34 and above
* for {@link AsynchronousMediaCodecAdapter} instances.
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*/
@CanIgnoreReturnValue
public DefaultMediaCodecAdapterFactory experimentalSetAsyncCryptoFlagEnabled(
boolean enableAsyncCryptoFlag) {
asyncCryptoFlagEnabled = enableAsyncCryptoFlag;
return this;
}

@Override
public MediaCodecAdapter createAdapter(MediaCodecAdapter.Configuration configuration)
throws IOException {
Expand All @@ -96,6 +113,7 @@ public MediaCodecAdapter createAdapter(MediaCodecAdapter.Configuration configura
+ Util.getTrackTypeString(trackType));
AsynchronousMediaCodecAdapter.Factory factory =
new AsynchronousMediaCodecAdapter.Factory(trackType);
factory.experimentalSetAsyncCryptoFlagEnabled(asyncCryptoFlagEnabled);
return factory.createAdapter(configuration);
}
return new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration);
Expand Down
Loading

0 comments on commit e5aa692

Please sign in to comment.