Merge Issue: #1826: add extension for MPEG-H decoding

Imported from GitHub PR #1826

Merge 6b59a16 into b5615d5

COPYBARA_INTEGRATE_REVIEW=#1826 from androidx:mpegh_extension 6b59a16
PiperOrigin-RevId: 689417378

RELEASENOTES.md

@@ -52,6 +52,9 @@
 *   Smooth Streaming Extension:
 *   RTSP Extension:
 *   Decoder Extensions (FFmpeg, VP9, AV1, etc.):
+    *   Add the MPEG-H decoder module which uses the native MPEG-H decoder
+        module to decode MPEG-H audio
+        ([#1826](https://github.com/androidx/media/pull/1826)).
 *   MIDI extension:
 *   Leanback extension:
 *   Cast Extension:

core_settings.gradle

@@ -83,6 +83,8 @@ if (gradle.ext.has('androidxMediaEnableMidiModule') && gradle.ext.androidxMediaE
     include modulePrefix + 'lib-decoder-midi'
     project(modulePrefix + 'lib-decoder-midi').projectDir = new File(rootDir, 'libraries/decoder_midi')
 }
+include modulePrefix + 'lib-decoder-mpegh'
+project(modulePrefix + 'lib-decoder-mpegh').projectDir = new File(rootDir, 'libraries/decoder_mpegh')
 include modulePrefix + 'lib-decoder-opus'
 project(modulePrefix + 'lib-decoder-opus').projectDir = new File(rootDir, 'libraries/decoder_opus')
 include modulePrefix + 'lib-decoder-vp9'

demos/main/build.gradle

@@ -90,6 +90,7 @@ dependencies {
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-iamf')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-vp9')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-midi')
+    withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-mpegh')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-datasource-rtmp')
 }
 

demos/main/src/main/assets/media.exolist.json

@@ -762,6 +762,10 @@
       {
         "name": "Immersive Audio Format Sample (MP4, IAMF)",
         "uri": "https://github.com/AOMediaCodec/libiamf/raw/main/tests/test_000036_s.mp4"
+      },
+      {
+        "name": "MPEG-H HD (MP4, H265)",
+        "uri": "https://media.githubusercontent.com/media/Fraunhofer-IIS/mpegh-test-content/main/TRI_Fileset_17_514H_D1_D2_D3_O1_24bit1080p50.mp4"
       }
     ]
   },

libraries/decoder_mpegh/README.md

@@ -0,0 +1,89 @@
+# MPEG-H decoder module
+
+The MPEG-H module provides `MpeghAudioRenderer`, which uses the libmpegh native
+library to decode MPEG-H audio.
+
+## License note
+
+Please note that while the code in this repository is licensed under
+[Apache 2.0][], using this module also requires building and including the
+[Fraunhofer GitHub MPEG-H decoder][] which is licensed under the Fraunhofer
+GitHub MPEG-H decoder project.
+
+[Apache 2.0]: ../../LICENSE
+[Fraunhofer GitHub MPEG-H decoder]: https://github.com/Fraunhofer-IIS/mpeghdec
+
+## Build instructions (Linux, macOS)
+
+To use the module you need to clone this GitHub project and depend on its
+modules locally. Instructions for doing this can be found in the
+[top level README][].
+
+In addition, it's necessary to fetch libmpegh library as follows:
+
+*   Set the following environment variables:
+
+```
+cd "<path to project checkout>"
+MPEGH_MODULE_PATH="$(pwd)/libraries/decoder_mpegh/src/main"
+```
+
+*   Fetch libmpegh library:
+
+```
+cd "${MPEGH_MODULE_PATH}/jni" && \
+git clone https://github.com/Fraunhofer-IIS/mpeghdec.git --branch r2.0.0 libmpegh
+```
+
+*   [Install CMake][].
+
+Having followed these steps, gradle will build the module automatically when run
+on the command line or via Android Studio, using [CMake][] and [Ninja][] to
+configure and build mpeghdec and the module's [JNI wrapper library][].
+
+[top level README]: ../../README.md
+[Install CMake]: https://developer.android.com/studio/projects/install-ndk
+[CMake]: https://cmake.org/
+[Ninja]: https://ninja-build.org
+[JNI wrapper library]: src/main/jni/mpeghdec_jni.cc
+
+## Build instructions (Windows)
+
+We do not provide support for building this module on Windows, however it should
+be possible to follow the Linux instructions in [Windows PowerShell][].
+
+[Windows PowerShell]: https://docs.microsoft.com/en-us/powershell/scripting/getting-started/getting-started-with-windows-powershell
+
+## Using the module with ExoPlayer
+
+Once you've followed the instructions above to check out, build and depend on
+the module, the next step is to tell ExoPlayer to use `MpeghAudioRenderer`. How
+you do this depends on which player API you're using:
+
+*   If you're passing a `DefaultRenderersFactory` to `ExoPlayer.Builder`, you
+    can enable using the module by setting the `extensionRendererMode` parameter
+    of the `DefaultRenderersFactory` constructor to
+    `EXTENSION_RENDERER_MODE_ON`. This will use `MpeghAudioRenderer` for
+    playback if `MediaCodecAudioRenderer` doesn't support the input format. Pass
+    `EXTENSION_RENDERER_MODE_PREFER` to give `MpeghAudioRenderer` priority over
+    `MediaCodecAudioRenderer`.
+*   If you've subclassed `DefaultRenderersFactory`, add a `MpeghAudioRenderer`
+    to the output list in `buildAudioRenderers`. ExoPlayer will use the first
+    `Renderer` in the list that supports the input media format.
+*   If you've implemented your own `RenderersFactory`, return a
+    `MpeghAudioRenderer` instance from `createRenderers`. ExoPlayer will use the
+    first `Renderer` in the returned array that supports the input media format.
+*   If you're using `ExoPlayer.Builder`, pass a `MpeghAudioRenderer` in the
+    array of `Renderer`s. ExoPlayer will use the first `Renderer` in the list
+    that supports the input media format.
+
+Note: These instructions assume you're using `DefaultTrackSelector`. If you have
+a custom track selector the choice of `Renderer` is up to your implementation,
+so you need to make sure you are passing an `MpeghAudioRenderer` to the player,
+then implement your own logic to use the renderer for a given track.
+
+## Links
+
+*   [Troubleshooting using decoding extensions][]
+
+[Troubleshooting using decoding extensions]: https://developer.android.com/media/media3/exoplayer/troubleshooting#how-can-i-get-a-decoding-library-to-load-and-be-used-for-playback

libraries/decoder_mpegh/build.gradle

@@ -0,0 +1,65 @@
+// Copyright 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+apply from: "$gradle.ext.androidxMediaSettingsDir/common_library_config.gradle"
+
+android {
+    namespace 'androidx.media3.decoder.mpegh'
+
+    sourceSets {
+        androidTest.assets.srcDir '../test_data/src/test/assets'
+    }
+
+    defaultConfig {
+        externalNativeBuild {
+            cmake {
+                targets "mpeghJNI"
+            }
+        }
+    }
+
+    // TODO(Internal: b/372449691): Remove packagingOptions once AGP is updated
+    // to version 8.5.1 or higher.
+    packagingOptions {
+        jniLibs {
+            useLegacyPackaging true
+        }
+    }
+}
+
+// Configure the native build only if libmpegh is present to avoid gradle sync
+// failures if libmpegh hasn't been built according to the README instructions.
+if (project.file('src/main/jni/libmpegh').exists()) {
+    android.externalNativeBuild.cmake {
+        path = 'src/main/jni/CMakeLists.txt'
+        version = '3.21.0+'
+        if (project.hasProperty('externalNativeBuildDir')) {
+            if (!new File(externalNativeBuildDir).isAbsolute()) {
+                ext.externalNativeBuildDir =
+                        new File(rootDir, it.externalNativeBuildDir)
+            }
+            buildStagingDirectory = "${externalNativeBuildDir}/${project.name}"
+        }
+    }
+}
+
+dependencies {
+    implementation project(modulePrefix + 'lib-decoder')
+    // TODO(b/203752526): Remove this dependency.
+    implementation project(modulePrefix + 'lib-exoplayer')
+    implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
+    compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
+    compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
+    testImplementation project(modulePrefix + 'test-utils')
+    testImplementation 'org.robolectric:robolectric:' + robolectricVersion
+}

libraries/decoder_mpegh/proguard-rules.txt

@@ -0,0 +1,13 @@
+# Proguard rules specific to the MPEG-H extension.
+
+# This prevents the names of native methods from being obfuscated.
+-keepclasseswithmembernames class * {
+    native <methods>;
+}
+
+# Some members of this class are being accessed from native methods. Keep them unobfuscated.
+-keep class androidx.media3.decoder.SimpleDecoderOutputBuffer {
+    *;
+}
+
+-keep class androidx.media3.decoder.mpegh** { *; }

libraries/decoder_mpegh/src/main/AndroidManifest.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2024 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<manifest package="androidx.media3.decoder.mpegh"/>

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import android.os.Handler;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.audio.AudioProcessor;
+import androidx.media3.common.util.TraceUtil;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.CryptoConfig;
+import androidx.media3.exoplayer.DecoderReuseEvaluation;
+import androidx.media3.exoplayer.audio.AudioRendererEventListener;
+import androidx.media3.exoplayer.audio.AudioSink;
+import androidx.media3.exoplayer.audio.DecoderAudioRenderer;
+import java.util.Objects;
+
+/** Decodes and renders audio using the native MPEG-H decoder. */
+@UnstableApi
+public final class MpeghAudioRenderer extends DecoderAudioRenderer<MpeghDecoder> {
+
+  private static final String TAG = "MpeghAudioRenderer";
+
+  /** The number of input and output buffers. */
+  private static final int NUM_BUFFERS = 16;
+
+  /*  Creates a new instance. */
+  public MpeghAudioRenderer() {
+    this(/* eventHandler= */ null, /* eventListener= */ null);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
+   *     null if delivery of events is not required.
+   * @param eventListener A listener of events. May be null if delivery of events is not required.
+   * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output.
+   */
+  public MpeghAudioRenderer(
+      Handler eventHandler,
+      AudioRendererEventListener eventListener,
+      AudioProcessor... audioProcessors) {
+    super(eventHandler, eventListener, audioProcessors);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
+   *     null if delivery of events is not required.
+   * @param eventListener A listener of events. May be null if delivery of events is not required.
+   * @param audioSink The sink to which audio will be output.
+   */
+  public MpeghAudioRenderer(
+      Handler eventHandler, AudioRendererEventListener eventListener, AudioSink audioSink) {
+    super(eventHandler, eventListener, audioSink);
+  }
+
+  @Override
+  public String getName() {
+    return TAG;
+  }
+
+  @Override
+  protected @C.FormatSupport int supportsFormatInternal(Format format) {
+    // Check if JNI library is available.
+    if (!MpeghLibrary.isAvailable()) {
+      return C.FORMAT_UNSUPPORTED_TYPE;
+    }
+
+    // Check if MIME type is supported.
+    if (!(Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHM1)
+        || Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHA1))) {
+      return C.FORMAT_UNSUPPORTED_TYPE;
+    }
+    return C.FORMAT_HANDLED;
+  }
+
+  @Override
+  protected DecoderReuseEvaluation canReuseDecoder(
+      String decoderName, Format oldFormat, Format newFormat) {
+    if (Objects.equals(oldFormat.sampleMimeType, newFormat.sampleMimeType)
+        && Objects.equals(oldFormat.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHM1)) {
+      return new DecoderReuseEvaluation(
+          decoderName,
+          oldFormat,
+          newFormat,
+          DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
+          /* discardReasons= */ 0);
+    }
+    return super.canReuseDecoder(decoderName, oldFormat, newFormat);
+  }
+
+  @Override
+  protected MpeghDecoder createDecoder(Format format, CryptoConfig cryptoConfig)
+      throws MpeghDecoderException {
+    TraceUtil.beginSection("createMpeghDecoder");
+    MpeghDecoder decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS);
+    TraceUtil.endSection();
+    return decoder;
+  }
+
+  @Override
+  protected Format getOutputFormat(MpeghDecoder decoder) {
+    return new Format.Builder()
+        .setChannelCount(decoder.getChannelCount())
+        .setSampleRate(decoder.getSampleRate())
+        .setSampleMimeType(MimeTypes.AUDIO_RAW)
+        .setPcmEncoding(C.ENCODING_PCM_16BIT)
+        .build();
+  }
+}