Add experimental opt-in to parse DASH subtitles during extraction

This currently only applies to subtitles muxed into mp4 segments, and
not standalone text files linked directly from the manifest.

Issue: androidx/media#288

#minor-release

PiperOrigin-RevId: 572263764

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/BundledChunkExtractor.java

@@ -21,6 +21,7 @@
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.analytics.PlayerId;
 import com.google.android.exoplayer2.extractor.ChunkIndex;
 import com.google.android.exoplayer2.extractor.DummyTrackOutput;
 import com.google.android.exoplayer2.extractor.Extractor;
@@ -31,11 +32,14 @@
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.mkv.MatroskaExtractor;
 import com.google.android.exoplayer2.extractor.mp4.FragmentedMp4Extractor;
+import com.google.android.exoplayer2.text.SubtitleParser;
+import com.google.android.exoplayer2.text.SubtitleTranscodingExtractor;
 import com.google.android.exoplayer2.upstream.DataReader;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.ParsableByteArray;
 import java.io.IOException;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
@@ -50,36 +54,68 @@
 @Deprecated
 public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtractor {
 
-  /** {@link ChunkExtractor.Factory} for instances of this class. */
-  public static final ChunkExtractor.Factory FACTORY =
-      (primaryTrackType,
-          format,
-          enableEventMessageTrack,
-          closedCaptionFormats,
-          playerEmsgTrackOutput,
-          playerId) -> {
-        @Nullable String containerMimeType = format.containerMimeType;
-        Extractor extractor;
-        if (MimeTypes.isText(containerMimeType)) {
-          // Text types do not need an extractor.
-          return null;
-        } else if (MimeTypes.isMatroska(containerMimeType)) {
-          extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
-        } else {
-          int flags = 0;
-          if (enableEventMessageTrack) {
-            flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
-          }
-          extractor =
-              new FragmentedMp4Extractor(
-                  flags,
-                  /* timestampAdjuster= */ null,
-                  /* sideloadedTrack= */ null,
-                  closedCaptionFormats,
-                  playerEmsgTrackOutput);
+  /** {@link ChunkExtractor.Factory} for {@link BundledChunkExtractor}. */
+  public static final class Factory implements ChunkExtractor.Factory {
+
+    /** Non-null if subtitles should be parsed during extraction, null otherwise. */
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
+
+    /**
+     * Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or
+     * null to parse subtitles during decoding. The default is null (subtitles parsed after
+     * decoding).
+     *
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+     * in a future release.
+     *
+     * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+     *     extraction.
+     * @return This factory, for convenience.
+     */
+    public Factory experimentalSetSubtitleParserFactory(
+        @Nullable SubtitleParser.Factory subtitleParserFactory) {
+      this.subtitleParserFactory = subtitleParserFactory;
+      return this;
+    }
+
+    @Nullable
+    @Override
+    public ChunkExtractor createProgressiveMediaExtractor(
+        @C.TrackType int primaryTrackType,
+        Format representationFormat,
+        boolean enableEventMessageTrack,
+        List<Format> closedCaptionFormats,
+        @Nullable TrackOutput playerEmsgTrackOutput,
+        PlayerId playerId) {
+      @Nullable String containerMimeType = representationFormat.containerMimeType;
+      Extractor extractor;
+      if (MimeTypes.isText(containerMimeType)) {
+        // Text types do not need an extractor.
+        return null;
+      } else if (MimeTypes.isMatroska(containerMimeType)) {
+        extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
+      } else {
+        int flags = 0;
+        if (enableEventMessageTrack) {
+          flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
         }
-        return new BundledChunkExtractor(extractor, primaryTrackType, format);
-      };
+        extractor =
+            new FragmentedMp4Extractor(
+                flags,
+                /* timestampAdjuster= */ null,
+                /* sideloadedTrack= */ null,
+                closedCaptionFormats,
+                playerEmsgTrackOutput);
+      }
+      if (subtitleParserFactory != null) {
+        extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
+      }
+      return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);
+    }
+  }
+
+  /** {@link Factory} for {@link BundledChunkExtractor}. */
+  public static final Factory FACTORY = new Factory();
 
   private static final PositionHolder POSITION_HOLDER = new PositionHolder();
 

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java

@@ -49,6 +49,7 @@
 import com.google.android.exoplayer2.source.dash.manifest.EventStream;
 import com.google.android.exoplayer2.source.dash.manifest.Period;
 import com.google.android.exoplayer2.source.dash.manifest.Representation;
+import com.google.android.exoplayer2.text.SubtitleParser;
 import com.google.android.exoplayer2.trackselection.ExoTrackSelection;
 import com.google.android.exoplayer2.upstream.Allocator;
 import com.google.android.exoplayer2.upstream.CmcdConfiguration;
@@ -138,7 +139,8 @@ public DashMediaPeriod(
       Allocator allocator,
       CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
       PlayerEmsgCallback playerEmsgCallback,
-      PlayerId playerId) {
+      PlayerId playerId,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.id = id;
     this.manifest = manifest;
     this.baseUrlExclusionList = baseUrlExclusionList;
@@ -164,7 +166,8 @@ public DashMediaPeriod(
     Period period = manifest.getPeriod(periodIndex);
     eventStreams = period.eventStreams;
     Pair<TrackGroupArray, TrackGroupInfo[]> result =
-        buildTrackGroups(drmSessionManager, period.adaptationSets, eventStreams);
+        buildTrackGroups(
+            drmSessionManager, subtitleParserFactory, period.adaptationSets, eventStreams);
     trackGroups = result.first;
     trackGroupInfos = result.second;
   }
@@ -509,6 +512,7 @@ private int getPrimaryStreamIndex(int embeddedStreamIndex, int[] streamIndexToTr
 
   private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
       DrmSessionManager drmSessionManager,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       List<AdaptationSet> adaptationSets,
       List<EventStream> eventStreams) {
     int[][] groupedAdaptationSetIndices = getGroupedAdaptationSetIndices(adaptationSets);
@@ -531,6 +535,7 @@ private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
     int trackGroupCount =
         buildPrimaryAndEmbeddedTrackGroupInfos(
             drmSessionManager,
+            subtitleParserFactory,
             adaptationSets,
             groupedAdaptationSetIndices,
             primaryGroupCount,
@@ -670,6 +675,7 @@ private static int identifyEmbeddedTracks(
 
   private static int buildPrimaryAndEmbeddedTrackGroupInfos(
       DrmSessionManager drmSessionManager,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       List<AdaptationSet> adaptationSets,
       int[][] groupedAdaptationSetIndices,
       int primaryGroupCount,
@@ -686,8 +692,24 @@ private static int buildPrimaryAndEmbeddedTrackGroupInfos(
       }
       Format[] formats = new Format[representations.size()];
       for (int j = 0; j < formats.length; j++) {
-        Format format = representations.get(j).format;
-        formats[j] = format.copyWithCryptoType(drmSessionManager.getCryptoType(format));
+        Format originalFormat = representations.get(j).format;
+        Format.Builder updatedFormat =
+            originalFormat
+                .buildUpon()
+                .setCryptoType(drmSessionManager.getCryptoType(originalFormat));
+        if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(originalFormat)) {
+          updatedFormat
+              .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
+              .setCueReplacementBehavior(
+                  subtitleParserFactory.getCueReplacementBehavior(originalFormat))
+              .setCodecs(
+                  originalFormat.sampleMimeType
+                      + (originalFormat.codecs != null ? " " + originalFormat.codecs : ""))
+              // Reset this value to the default. All non-default timestamp adjustments are done
+              // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
+              .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
+        }
+        formats[j] = updatedFormat.build();
       }
 
       AdaptationSet firstAdaptationSet = adaptationSets.get(adaptationSetIndices[0]);

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaSource.java

@@ -60,6 +60,8 @@
 import com.google.android.exoplayer2.source.dash.manifest.Period;
 import com.google.android.exoplayer2.source.dash.manifest.Representation;
 import com.google.android.exoplayer2.source.dash.manifest.UtcTimingElement;
+import com.google.android.exoplayer2.text.DefaultSubtitleParserFactory;
+import com.google.android.exoplayer2.text.SubtitleParser;
 import com.google.android.exoplayer2.upstream.Allocator;
 import com.google.android.exoplayer2.upstream.CmcdConfiguration;
 import com.google.android.exoplayer2.upstream.DataSource;
@@ -118,6 +120,7 @@ public static final class Factory implements MediaSourceFactory {
     private DrmSessionManagerProvider drmSessionManagerProvider;
     private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
     private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
     private long fallbackTargetLiveOffsetMs;
     private long minLiveStartPositionUs;
     @Nullable private ParsingLoadable.Parser<? extends DashManifest> manifestParser;
@@ -202,6 +205,40 @@ public Factory setLoadErrorHandlingPolicy(LoadErrorHandlingPolicy loadErrorHandl
       return this;
     }
 
+    /**
+     * Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
+     * part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
+     * as part of rendering).
+     *
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+     * in a future release.
+     *
+     * <p>This method may only be used with {@link DefaultDashChunkSource.Factory}.
+     *
+     * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
+     *     rendering.
+     * @return This factory, for convenience.
+     */
+    // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
+    //  DefaultMediaSourceFactory via the MediaSource.Factory interface).
+    public Factory experimentalParseSubtitlesDuringExtraction(
+        boolean parseSubtitlesDuringExtraction) {
+      if (parseSubtitlesDuringExtraction) {
+        if (subtitleParserFactory == null) {
+          this.subtitleParserFactory = new DefaultSubtitleParserFactory();
+        }
+      } else {
+        this.subtitleParserFactory = null;
+      }
+      if (chunkSourceFactory instanceof DefaultDashChunkSource.Factory) {
+        ((DefaultDashChunkSource.Factory) chunkSourceFactory)
+            .setSubtitleParserFactory(subtitleParserFactory);
+      } else {
+        throw new IllegalStateException();
+      }
+      return this;
+    }
+
     /**
      * Sets the target {@link Player#getCurrentLiveOffset() offset for live streams} that is used if
      * no value is defined in the {@link MediaItem} or the manifest.
@@ -321,6 +358,7 @@ public DashMediaSource createMediaSource(DashManifest manifest, MediaItem mediaI
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           fallbackTargetLiveOffsetMs,
           minLiveStartPositionUs);
     }
@@ -359,6 +397,7 @@ public DashMediaSource createMediaSource(MediaItem mediaItem) {
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           fallbackTargetLiveOffsetMs,
           minLiveStartPositionUs);
     }
@@ -417,6 +456,7 @@ public DashMediaSource createMediaSource(MediaItem mediaItem) {
   private final Runnable simulateManifestRefreshRunnable;
   private final PlayerEmsgCallback playerEmsgCallback;
   private final LoaderErrorThrower manifestLoadErrorThrower;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   private DataSource dataSource;
   private Loader loader;
@@ -452,6 +492,7 @@ private DashMediaSource(
       @Nullable CmcdConfiguration cmcdConfiguration,
       DrmSessionManager drmSessionManager,
       LoadErrorHandlingPolicy loadErrorHandlingPolicy,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       long fallbackTargetLiveOffsetMs,
       long minLiveStartPositionUs) {
     this.mediaItem = mediaItem;
@@ -465,6 +506,7 @@ private DashMediaSource(
     this.cmcdConfiguration = cmcdConfiguration;
     this.drmSessionManager = drmSessionManager;
     this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
+    this.subtitleParserFactory = subtitleParserFactory;
     this.fallbackTargetLiveOffsetMs = fallbackTargetLiveOffsetMs;
     this.minLiveStartPositionUs = minLiveStartPositionUs;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
@@ -570,7 +612,8 @@ public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long star
             allocator,
             compositeSequenceableLoaderFactory,
             playerEmsgCallback,
-            getPlayerId());
+            getPlayerId(),
+            subtitleParserFactory);
     periodsById.put(mediaPeriod.id, mediaPeriod);
     return mediaPeriod;
   }

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java

@@ -48,6 +48,7 @@
 import com.google.android.exoplayer2.source.dash.manifest.DashManifest;
 import com.google.android.exoplayer2.source.dash.manifest.RangedUri;
 import com.google.android.exoplayer2.source.dash.manifest.Representation;
+import com.google.android.exoplayer2.text.SubtitleParser;
 import com.google.android.exoplayer2.trackselection.ExoTrackSelection;
 import com.google.android.exoplayer2.upstream.CmcdConfiguration;
 import com.google.android.exoplayer2.upstream.CmcdData;
@@ -77,6 +78,7 @@
 @Deprecated
 public class DefaultDashChunkSource implements DashChunkSource {
 
+  /** {@link DashChunkSource.Factory} for {@link DefaultDashChunkSource} instances. */
   public static final class Factory implements DashChunkSource.Factory {
 
     private final DataSource.Factory dataSourceFactory;
@@ -116,6 +118,21 @@ public Factory(
       this.maxSegmentsPerLoad = maxSegmentsPerLoad;
     }
 
+    /**
+     * Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
+     * or null to parse subtitles during decoding.
+     *
+     * <p>This may only be used with {@link BundledChunkExtractor.Factory}.
+     */
+    /* package */ Factory setSubtitleParserFactory(
+        @Nullable SubtitleParser.Factory subtitleParserFactory) {
+      if (chunkExtractorFactory instanceof BundledChunkExtractor.Factory) {
+        ((BundledChunkExtractor.Factory) chunkExtractorFactory)
+            .experimentalSetSubtitleParserFactory(subtitleParserFactory);
+      }
+      return this;
+    }
+
     @Override
     public DashChunkSource createDashChunkSource(
         LoaderErrorThrower manifestLoaderErrorThrower,

library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java

@@ -224,7 +224,8 @@ private static DashMediaPeriod createDashMediaPeriod(DashManifest manifest, int
         mock(Allocator.class),
         mock(CompositeSequenceableLoaderFactory.class),
         mock(PlayerEmsgCallback.class),
-        PlayerId.UNSET);
+        PlayerId.UNSET,
+        /* subtitleParserFactory= */ null);
   }
 
   private static DashManifest parseManifest(String fileName) throws IOException {

library/dash/src/test/java/com/google/android/exoplayer2/source/dash/e2etest/DashPlaybackTest.java

@@ -32,10 +32,12 @@
 import com.google.android.exoplayer2.robolectric.PlaybackOutput;
 import com.google.android.exoplayer2.robolectric.ShadowMediaCodecConfig;
 import com.google.android.exoplayer2.robolectric.TestPlayerRunHelper;
+import com.google.android.exoplayer2.source.dash.DashMediaSource;
 import com.google.android.exoplayer2.testutil.CapturingRenderersFactory;
 import com.google.android.exoplayer2.testutil.DumpFileAsserts;
 import com.google.android.exoplayer2.testutil.FakeClock;
 import com.google.android.exoplayer2.trackselection.DefaultTrackSelector;
+import com.google.android.exoplayer2.upstream.DefaultDataSource;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -79,14 +81,15 @@ public void ttmlStandaloneXmlFile() throws Exception {
 
   // https://github.com/google/ExoPlayer/issues/7985
   @Test
-  @Ignore(
-      "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
   public void webvttInMp4() throws Exception {
     Context applicationContext = ApplicationProvider.getApplicationContext();
     CapturingRenderersFactory capturingRenderersFactory =
         new CapturingRenderersFactory(applicationContext);
     ExoPlayer player =
         new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setMediaSourceFactory(
+                new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
+                    .experimentalParseSubtitlesDuringExtraction(true))
             .setClock(new FakeClock(/* isAutoAdvancing= */ true))
             .build();
     player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));