From 940854c86ff500317a5f5934cb8574a05e4e0b53 Mon Sep 17 00:00:00 2001
From: nanahi <130121847+na-na-hi@users.noreply.github.com>
Date: Fri, 21 Jun 2024 03:43:08 -0400
Subject: [PATCH] demux: allow refresh seek for non-video streams

Since d41f0a54b07d855227b63a6b2c4e92794ff7b86f, enabling a stream will
no longer perform a refresh seek, to fix issues with switching video
tracks. Additionally, 62e9a0c5f6be63c4cbe6387cbd3419fe19e98b74 also
prevents refresh seek on track switching when it happens right after a
seek.

Unfortunately, when external audio files are loaded, preventing refresh
seeks can cause A-V sync issues. Since external tracks have separate demuxer
instances from the internal tracks, the demuxer instances from both
internal and external tracks are explicitly sought to the same pts when
seeking is performed. When enabling an external audio track for the first
time, the existing logic prevents refresh seek because no packets have ever
been read, so the track ends up not being sought. This means that switching
the track in the middle of playback results in a huge A-V desync.

To make it worse, unlike one demuxer instance with multiple tracks,
tracks from multiple demuxer instances are not synced in any meaningful
way beyond the separate explicit seeking. The only thing which keeps these
tracks synced is the generic A-V sync logic, which is unfit for such large
desync above. This means the audio is at the start position after the track
is switched, behind the video, so the audio is not considered ready, and
audio is continuously filtered sequentially until it is synced to the video.
While this is happening, the audio filtering exhausts all CPU resources.

There is also a case with cache enabled: if a seek causes some new data to
be cached, the demuxer is sought to the end of joined cache range.
If track is switched after this, the existing logic prevents a refresh seek,
so the demuxer is at a wrong position after the track is switched.

Fix this by allowing refresh seek for non-video streams, even when no packets
have been read yet or immediately after a seek.
---
 demux/demux.c | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/demux/demux.c b/demux/demux.c
index 6feb97e9987af..d31899b3d9a3c 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -3964,6 +3964,28 @@ static void initiate_refresh_seek(struct demux_internal *in,
     in->seek_pts = start_ts;
 }
 
+// Called locked.
+static void refresh_track(struct demuxer *demuxer, struct sh_stream *stream,
+                          double ref_pts)
+{
+    struct demux_internal *in = demuxer->in;
+    struct demux_stream *ds = stream->ds;
+    ref_pts = MP_ADD_PTS(ref_pts, -in->ts_offset);
+
+    MP_VERBOSE(in, "refresh track %d\n", stream->index);
+    if (in->back_demuxing)
+        ds->back_seek_pos = ref_pts;
+    // Allow refresh seek for non-video streams, even if no packets have
+    // ever been read yet or after seeking. This is necessary because:
+    // - A-V sync targets may come from different demuxers, so enabling an external
+    //   audio track for the first time can cause desync since a seek is not performed.
+    // - If cache is enabled and a seek causes some new data to be cached, the demuxer
+    //   is sought to the end of cache after cache joining. Switching track immediately
+    //   after this also causes the same problem.
+    if (!in->after_seek || ds->type != STREAM_VIDEO)
+        initiate_refresh_seek(in, ds, ref_pts);
+}
+
 // Set whether the given stream should return packets.
 // ref_pts is used only if the stream is enabled. Then it serves as approximate
 // start pts for this stream (in the worst case it is ignored).
@@ -3973,19 +3995,14 @@ void demuxer_select_track(struct demuxer *demuxer, struct sh_stream *stream,
     struct demux_internal *in = demuxer->in;
     struct demux_stream *ds = stream->ds;
     mp_mutex_lock(&in->lock);
-    ref_pts = MP_ADD_PTS(ref_pts, -in->ts_offset);
     // don't flush buffers if stream is already selected / unselected
     if (ds->selected != selected) {
         MP_VERBOSE(in, "%sselect track %d\n", selected ? "" : "de", stream->index);
         ds->selected = selected;
         update_stream_selection_state(in, ds);
         in->tracks_switched = true;
-        if (ds->selected) {
-            if (in->back_demuxing)
-                ds->back_seek_pos = ref_pts;
-            if (!in->after_seek)
-                initiate_refresh_seek(in, ds, ref_pts);
-        }
+        if (ds->selected)
+            refresh_track(demuxer, stream, ref_pts);
         if (in->threading) {
             mp_cond_signal(&in->wakeup);
         } else {
@@ -4003,14 +4020,9 @@ void demuxer_refresh_track(struct demuxer *demuxer, struct sh_stream *stream,
     struct demux_internal *in = demuxer->in;
     struct demux_stream *ds = stream->ds;
     mp_mutex_lock(&in->lock);
-    ref_pts = MP_ADD_PTS(ref_pts, -in->ts_offset);
     if (ds->selected) {
-        MP_VERBOSE(in, "refresh track %d\n", stream->index);
         update_stream_selection_state(in, ds);
-        if (in->back_demuxing)
-            ds->back_seek_pos = ref_pts;
-        if (!in->after_seek)
-            initiate_refresh_seek(in, ds, ref_pts);
+        refresh_track(demuxer, stream, ref_pts);
     }
     mp_mutex_unlock(&in->lock);
 }