From 04196b3b68bfbe5160eaa051910bfadcfb33d3dd Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Sun, 21 Feb 2021 16:28:29 +0400
Subject: [PATCH 001/394] [StreamwoIE] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamwo.py   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/streamwo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1a39c25c52d..88c3dcb9ecc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1159,6 +1159,7 @@
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streamwo import StreamwoIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
new file mode 100644
index 00000000000..74ed3025261
--- /dev/null
+++ b/youtube_dl/extractor/streamwo.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamwoIE(InfoExtractor):
+    _VALID_URL = r'https?://streamwo\.com/(?P<id>\w+)'
+    _TEST = {
+        "url": 'https://streamwo.com/4529aff4',
+        'md5': 'eaf1f163635c868ecbba95d23ba83448',
+        'info_dict': {
+            'id': '4529aff4',
+            'ext': 'mp4',
+            'title': 'Goal 2021 13-04-50'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        self.report_extraction(video_id)
+        title = self._html_search_regex(r'<span class="titles">(.+?)</span>', webpage, 'title')
+        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            "title": title
+        }

From dd640efb4f45a72c46212b9ecf539eb18ebc070a Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sun, 21 Feb 2021 18:17:28 +0400
Subject: [PATCH 002/394] Update streamwo.py

---
 youtube_dl/extractor/streamwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
index 74ed3025261..db622901a55 100644
--- a/youtube_dl/extractor/streamwo.py
+++ b/youtube_dl/extractor/streamwo.py
@@ -21,7 +21,7 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, video_id)
         self.report_extraction(video_id)
         title = self._html_search_regex(r'<span class="titles">(.+?)</span>', webpage, 'title')
-        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
+        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
         return {
             'id': video_id,
             'url': video_url,

From 7a0e03ef28501a5a49a34d0a1dde23f2e11f6788 Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Mon, 22 Feb 2021 21:02:06 +0400
Subject: [PATCH 003/394] Replaced test video in streamwo.py

This test video should play fine in browser.
---
 youtube_dl/extractor/streamwo.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
index db622901a55..cf1d37d2517 100644
--- a/youtube_dl/extractor/streamwo.py
+++ b/youtube_dl/extractor/streamwo.py
@@ -7,12 +7,12 @@
 class StreamwoIE(InfoExtractor):
     _VALID_URL = r'https?://streamwo\.com/(?P<id>\w+)'
     _TEST = {
-        "url": 'https://streamwo.com/4529aff4',
-        'md5': 'eaf1f163635c868ecbba95d23ba83448',
+        "url": 'https://streamwo.com/c11cf208',
+        'md5': '64a3e444e10f90051725cc1776682b06',
         'info_dict': {
-            'id': '4529aff4',
+            'id': 'c11cf208',
             'ext': 'mp4',
-            'title': 'Goal 2021 13-04-50'
+            'title': 'pexels-rodnae-productions-6192787
         }
     }
 

From 236fc0d61b89f7dad8e35009739dd8d838720922 Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Mon, 22 Feb 2021 22:04:27 +0400
Subject: [PATCH 004/394] Update streamwo.py

Added missing '
---
 youtube_dl/extractor/streamwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
index cf1d37d2517..ec5ecd750d9 100644
--- a/youtube_dl/extractor/streamwo.py
+++ b/youtube_dl/extractor/streamwo.py
@@ -12,7 +12,7 @@ class StreamwoIE(InfoExtractor):
         'info_dict': {
             'id': 'c11cf208',
             'ext': 'mp4',
-            'title': 'pexels-rodnae-productions-6192787
+            'title': 'pexels-rodnae-productions-6192787'
         }
     }
 

From 250ee17e01062291c71f6c6ea3fda42a651bff3e Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Tue, 23 Feb 2021 10:50:29 +0400
Subject: [PATCH 005/394] updated Streamwo to update _real_extract method to
 return self.url_result of the embedded video

---
 youtube_dl/extractor/streamwo.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
index 74ed3025261..3edc296ad76 100644
--- a/youtube_dl/extractor/streamwo.py
+++ b/youtube_dl/extractor/streamwo.py
@@ -7,12 +7,12 @@
 class StreamwoIE(InfoExtractor):
     _VALID_URL = r'https?://streamwo\.com/(?P<id>\w+)'
     _TEST = {
-        "url": 'https://streamwo.com/4529aff4',
-        'md5': 'eaf1f163635c868ecbba95d23ba83448',
+        "url": 'https://streamwo.com/c11cf208',
+        'md5': '64a3e444e10f90051725cc1776682b06',
         'info_dict': {
-            'id': '4529aff4',
+            'id': 'zrxKAY8',
             'ext': 'mp4',
-            'title': 'Goal 2021 13-04-50'
+            'title': "Imgur"
         }
     }
 
@@ -20,11 +20,7 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         self.report_extraction(video_id)
-        title = self._html_search_regex(r'<span class="titles">(.+?)</span>', webpage, 'title')
-        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            "title": title
-        }
+        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
+        if video_url:
+            return self.url_result(video_url, ie="Generic")
+        return self.url_result(url, ie='Generic')

From 3c75b8dc156073f407ab102dca45659609cb32e9 Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Fri, 26 Feb 2021 14:09:36 +0400
Subject: [PATCH 006/394] Implemented get_referrer_url method in utils.py along
 with relevant tests in test_utils.py

---
 test/test_utils.py                 | 31 ++++++++++++++++++
 youtube_dl/extractor/common.py     | 16 ++++++++-
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/generic.py    | 10 ++++++
 youtube_dl/extractor/streamwo.py   | 26 ---------------
 youtube_dl/utils.py                | 52 ++++++++++++++++++++++++++++++
 6 files changed, 108 insertions(+), 28 deletions(-)
 delete mode 100644 youtube_dl/extractor/streamwo.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e1e..c867d72bba6 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -80,6 +80,7 @@
     subtitles_filename,
     timeconvert,
     unescapeHTML,
+    get_referrer_url,
     unified_strdate,
     unified_timestamp,
     unsmuggle_url,
@@ -306,6 +307,36 @@ def test_unescape_html(self):
         # HTML5 entities
         self.assertEqual(unescapeHTML('&period;&apos;'), '.\'')
 
+    def test_get_referrer_url(self):
+        # No-Referrer
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "no-referrer"), None)
+        # No-Referrer-When-Downgrade
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "no-referrer-when-downgrade"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "no-referrer-when-downgrade"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "no-referrer-when-downgrade"), None)
+        # Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "origin"), "https://example.com/")
+        # Origin-When-Cross-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "origin-when-cross-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://mozilla.org", "origin-when-cross-origin"), "https://example.com/")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "origin-when-cross-origin"), "https://example.com/")
+        # Same-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "same-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "same-origin"), None)
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "same-origin"), None)
+        # Strict-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin"), "https://example.com/")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "strict-origin"), None)
+        self.assertEqual(get_referrer_url("http://example.com/page", "http://example.com", "strict-origin"), "http://example.com/")
+        # Strict-Origin-When-Cross-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "strict-origin-when-cross-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin-when-cross-origin"), "https://example.com/")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/otherpage", "strict-origin-when-cross-origin"), None)
+        # Unsafe-Url
+        self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
+        self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://mozilla.org", "unsafe-url"), "https://example.com/page?q=123")
+        self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
+
     def test_date_from_str(self):
         self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
         self.assertEqual(date_from_str('now+7day'), date_from_str('now+1week'))
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8eb110f4e66..498f5e2138e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -81,6 +81,7 @@
     xpath_element,
     xpath_text,
     xpath_with_ns,
+    get_referrer_url
 )
 
 
@@ -2487,6 +2488,14 @@ def parse_content_type(content_type):
                 return f
             return {}
 
+        def get_referrer_policy(page):
+            mobj = re.search(r'<meta name="referrer" content="(.+?)">', page)
+            if mobj:
+                return mobj.group(1)
+            else:
+                # If no policy is set, default value is no-refferer-when-downgrade
+                return "no-referrer-when-downgrade"
+
         def _media_formats(src, cur_media_type, type_info={}):
             full_url = absolute_url(src)
             ext = type_info.get('ext') or determine_ext(full_url)
@@ -2509,6 +2518,7 @@ def _media_formats(src, cur_media_type, type_info={}):
             return is_plain_url, formats
 
         entries = []
+        referrer_policy = get_referrer_policy(webpage)
         # amp-video and amp-audio are very similar to their HTML5 counterparts
         # so we wll include them right here (see
         # https://www.ampproject.org/docs/reference/components/amp-video)
@@ -2590,7 +2600,11 @@ def _media_formats(src, cur_media_type, type_info={}):
                             'url': absolute_url(src),
                         })
             for f in media_info['formats']:
-                f.setdefault('http_headers', {})['Referer'] = base_url
+                referrer = get_referrer_url(base_url, f["url"], referrer_policy)
+                if referrer:
+                    f.setdefault('http_headers', {})['Referer'] = referrer
+                else:
+                    f.setdefault('http_headers', {})
             if media_info['formats'] or media_info['subtitles']:
                 entries.append(media_info)
         return entries
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 88c3dcb9ecc..1a39c25c52d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1159,7 +1159,6 @@
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
-from .streamwo import StreamwoIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c2b1b3bdfef..a1fc04375e6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2248,6 +2248,16 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 52,
         },
+        {
+            # Test StreamWo
+            "url": 'https://streamwo.com/c11cf208',
+            'md5': '64a3e444e10f90051725cc1776682b06',
+            'info_dict': {
+                'id': 'zrxKAY8',
+                'ext': 'mp4',
+                'title': "Imgur"
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
deleted file mode 100644
index 3edc296ad76..00000000000
--- a/youtube_dl/extractor/streamwo.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class StreamwoIE(InfoExtractor):
-    _VALID_URL = r'https?://streamwo\.com/(?P<id>\w+)'
-    _TEST = {
-        "url": 'https://streamwo.com/c11cf208',
-        'md5': '64a3e444e10f90051725cc1776682b06',
-        'info_dict': {
-            'id': 'zrxKAY8',
-            'ext': 'mp4',
-            'title': "Imgur"
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        self.report_extraction(video_id)
-        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
-        if video_url:
-            return self.url_result(video_url, ie="Generic")
-        return self.url_result(url, ie='Generic')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8e4d144c9ac..ba6e32f1873 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2326,6 +2326,58 @@ def bug_reports_message():
     return msg
 
 
+def get_referrer_url(base_url, target_url, policy):
+    # Returns correct referrer url based on the site policy
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#examples
+    parsed_url = compat_urlparse.urlparse(base_url)
+    parsed_target_url = compat_urlparse.urlparse(target_url)
+    if policy == "no-referrer":
+        return None
+    elif policy == "no-referrer-when-downgrade":
+        if "http" == parsed_target_url.scheme and "https" == parsed_url.scheme:
+            return None
+        elif "https" == parsed_target_url.scheme:
+            return base_url
+    elif policy == "origin":
+        return compat_urlparse.urljoin(base_url, "/")
+    elif policy == "origin-when-cross-origin":
+        if parsed_url.netloc == parsed_target_url.netloc:
+            if parsed_url.scheme == parsed_target_url.scheme:
+                return base_url
+            elif "http" == parsed_target_url.scheme:
+                return compat_urlparse.urljoin(base_url, "/")
+        else:
+            return compat_urlparse.urljoin(base_url, "/")
+    elif policy == "same-origin":
+        if parsed_url.netloc == parsed_target_url.netloc:
+            if parsed_url.scheme == parsed_target_url.scheme:
+                return base_url
+            else:
+                return None
+        else:
+            return None
+    elif policy == "strict-origin":
+        if "http" == parsed_url.scheme:
+            return compat_urlparse.urljoin(base_url, "/")
+        elif parsed_url.netloc == parsed_target_url.netloc:
+            if parsed_url.scheme == parsed_target_url.scheme:
+                return base_url
+            else:
+                return None
+        else:
+            return compat_urlparse.urljoin(base_url, "/")
+    elif policy == "strict-origin-when-cross-origin":
+        if parsed_url.netloc == parsed_target_url.netloc:
+            if parsed_url.scheme == parsed_target_url.scheme:
+                return base_url
+            else:
+                return None
+        else:
+            return compat_urlparse.urljoin(base_url, "/")
+    elif policy == "unsafe-url":
+        return base_url
+
+
 class YoutubeDLError(Exception):
     """Base exception for YoutubeDL errors."""
     pass

From b3c422a6a67fd8ff7493e7c8d2c210a6678d488b Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Mon, 1 Mar 2021 13:23:52 +0400
Subject: [PATCH 007/394] Implemented requested changes, added test cases to
 test_utils for the default fallback option

---
 test/test_utils.py              |  4 ++
 youtube_dl/extractor/common.py  | 16 ++-----
 youtube_dl/extractor/generic.py |  2 +-
 youtube_dl/utils.py             | 77 +++++++++++++++++----------------
 4 files changed, 48 insertions(+), 51 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index c867d72bba6..26743a813b4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -336,6 +336,10 @@ def test_get_referrer_url(self):
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://mozilla.org", "unsafe-url"), "https://example.com/page?q=123")
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
+        # None
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", None), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", None), "https://example.com/")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/otherpage", None), None)
 
     def test_date_from_str(self):
         self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 498f5e2138e..0259608db7e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2488,13 +2488,8 @@ def parse_content_type(content_type):
                 return f
             return {}
 
-        def get_referrer_policy(page):
-            mobj = re.search(r'<meta name="referrer" content="(.+?)">', page)
-            if mobj:
-                return mobj.group(1)
-            else:
-                # If no policy is set, default value is no-refferer-when-downgrade
-                return "no-referrer-when-downgrade"
+        def get_referrer_policy_from_meta_element(page):
+            return self._html_search_meta('referrer', page)
 
         def _media_formats(src, cur_media_type, type_info={}):
             full_url = absolute_url(src)
@@ -2518,7 +2513,7 @@ def _media_formats(src, cur_media_type, type_info={}):
             return is_plain_url, formats
 
         entries = []
-        referrer_policy = get_referrer_policy(webpage)
+        referrer_policy = get_referrer_policy_from_meta_element(webpage)
         # amp-video and amp-audio are very similar to their HTML5 counterparts
         # so we wll include them right here (see
         # https://www.ampproject.org/docs/reference/components/amp-video)
@@ -2601,10 +2596,7 @@ def _media_formats(src, cur_media_type, type_info={}):
                         })
             for f in media_info['formats']:
                 referrer = get_referrer_url(base_url, f["url"], referrer_policy)
-                if referrer:
-                    f.setdefault('http_headers', {})['Referer'] = referrer
-                else:
-                    f.setdefault('http_headers', {})
+                f.setdefault('http_headers', {})['Referer'] = referrer
             if media_info['formats'] or media_info['subtitles']:
                 entries.append(media_info)
         return entries
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a1fc04375e6..b1803372813 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2249,7 +2249,7 @@ class GenericIE(InfoExtractor):
             'playlist_mincount': 52,
         },
         {
-            # Test StreamWo
+            # Test Native <video> embed and application of the correct referrer url
             "url": 'https://streamwo.com/c11cf208',
             'md5': '64a3e444e10f90051725cc1776682b06',
             'info_dict': {
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ba6e32f1873..08a5edae507 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2326,56 +2326,57 @@ def bug_reports_message():
     return msg
 
 
-def get_referrer_url(base_url, target_url, policy):
+def get_referrer_url(referrer_source, request_url, referrer_policy):
     # Returns correct referrer url based on the site policy
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#examples
-    parsed_url = compat_urlparse.urlparse(base_url)
-    parsed_target_url = compat_urlparse.urlparse(target_url)
-    if policy == "no-referrer":
+    parsed_referrer_source = compat_urllib_parse_urlparse(referrer_source)
+    parsed_request_url = compat_urllib_parse_urlparse(request_url)
+    if referrer_policy == "no-referrer":
         return None
-    elif policy == "no-referrer-when-downgrade":
-        if "http" == parsed_target_url.scheme and "https" == parsed_url.scheme:
+    elif referrer_policy == "no-referrer-when-downgrade":
+        if "http" == parsed_request_url.scheme and "https" == parsed_referrer_source.scheme:
             return None
-        elif "https" == parsed_target_url.scheme:
-            return base_url
-    elif policy == "origin":
-        return compat_urlparse.urljoin(base_url, "/")
-    elif policy == "origin-when-cross-origin":
-        if parsed_url.netloc == parsed_target_url.netloc:
-            if parsed_url.scheme == parsed_target_url.scheme:
-                return base_url
-            elif "http" == parsed_target_url.scheme:
-                return compat_urlparse.urljoin(base_url, "/")
+        elif "https" == parsed_request_url.scheme:
+            return referrer_source
+    elif referrer_policy == "origin":
+        return compat_urlparse.urljoin(referrer_source, "/")
+    elif referrer_policy == "origin-when-cross-origin":
+        if parsed_referrer_source.netloc == parsed_request_url.netloc:
+            if parsed_referrer_source.scheme == parsed_request_url.scheme:
+                return referrer_source
+            elif "http" == parsed_request_url.scheme:
+                return compat_urlparse.urljoin(referrer_source, "/")
         else:
-            return compat_urlparse.urljoin(base_url, "/")
-    elif policy == "same-origin":
-        if parsed_url.netloc == parsed_target_url.netloc:
-            if parsed_url.scheme == parsed_target_url.scheme:
-                return base_url
+            return compat_urlparse.urljoin(referrer_source, "/")
+    elif referrer_policy == "same-origin":
+        if parsed_referrer_source.netloc == parsed_request_url.netloc:
+            if parsed_referrer_source.scheme == parsed_request_url.scheme:
+                return referrer_source
             else:
                 return None
         else:
             return None
-    elif policy == "strict-origin":
-        if "http" == parsed_url.scheme:
-            return compat_urlparse.urljoin(base_url, "/")
-        elif parsed_url.netloc == parsed_target_url.netloc:
-            if parsed_url.scheme == parsed_target_url.scheme:
-                return base_url
+    elif referrer_policy == "strict-origin":
+        if "http" == parsed_referrer_source.scheme:
+            return compat_urlparse.urljoin(referrer_source, "/")
+        elif parsed_referrer_source.netloc == parsed_request_url.netloc:
+            if parsed_referrer_source.scheme == parsed_request_url.scheme:
+                return referrer_source
             else:
                 return None
         else:
-            return compat_urlparse.urljoin(base_url, "/")
-    elif policy == "strict-origin-when-cross-origin":
-        if parsed_url.netloc == parsed_target_url.netloc:
-            if parsed_url.scheme == parsed_target_url.scheme:
-                return base_url
+            return compat_urlparse.urljoin(referrer_source, "/")
+    elif referrer_policy == "strict-origin-when-cross-origin" or referrer_policy is None:
+        # This is the default fallback if no referrer policy is defined
+        if parsed_referrer_source.netloc == parsed_request_url.netloc:
+            if parsed_referrer_source.scheme == parsed_request_url.scheme:
+                return referrer_source
             else:
                 return None
         else:
-            return compat_urlparse.urljoin(base_url, "/")
-    elif policy == "unsafe-url":
-        return base_url
+            return compat_urlparse.urljoin(referrer_source, "/")
+    elif referrer_policy == "unsafe-url":
+        return referrer_source
 
 
 class YoutubeDLError(Exception):
@@ -3953,10 +3954,10 @@ def urlencode_postdata(*args, **kargs):
 def update_url_query(url, query):
     if not query:
         return url
-    parsed_url = compat_urlparse.urlparse(url)
-    qs = compat_parse_qs(parsed_url.query)
+    parsed_referrer_source = compat_urlparse.urlparse(url)
+    qs = compat_parse_qs(parsed_referrer_source.query)
     qs.update(query)
-    return compat_urlparse.urlunparse(parsed_url._replace(
+    return compat_urlparse.urlunparse(parsed_referrer_source._replace(
         query=compat_urllib_parse_urlencode(qs, True)))
 
 

From 4c29d1017bf624cda0f80d49aada742bf477f49b Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Mon, 1 Mar 2021 17:34:18 +0400
Subject: [PATCH 008/394] Updated to only apply referrer if value is not None.
 Reverted back changes to utils.py

---
 youtube_dl/extractor/common.py | 3 ++-
 youtube_dl/utils.py            | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0259608db7e..be2b2549c59 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2596,7 +2596,8 @@ def _media_formats(src, cur_media_type, type_info={}):
                         })
             for f in media_info['formats']:
                 referrer = get_referrer_url(base_url, f["url"], referrer_policy)
-                f.setdefault('http_headers', {})['Referer'] = referrer
+                if referrer:
+                    f.setdefault('http_headers', {})['Referer'] = referrer
             if media_info['formats'] or media_info['subtitles']:
                 entries.append(media_info)
         return entries
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 08a5edae507..bb4c94c1658 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3954,10 +3954,10 @@ def urlencode_postdata(*args, **kargs):
 def update_url_query(url, query):
     if not query:
         return url
-    parsed_referrer_source = compat_urlparse.urlparse(url)
-    qs = compat_parse_qs(parsed_referrer_source.query)
+    parsed_url = compat_urlparse.urlparse(url)
+    qs = compat_parse_qs(parsed_url.query)
     qs.update(query)
-    return compat_urlparse.urlunparse(parsed_referrer_source._replace(
+    return compat_urlparse.urlunparse(parsed_url._replace(
         query=compat_urllib_parse_urlencode(qs, True)))
 
 

From 47b5b5586f606ed7f23ee7a98d04ae85936fe34e Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 1 Jun 2021 19:33:45 +0200
Subject: [PATCH 009/394] Added support to ThisVid.com

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/thisvid.py    | 97 ++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+)
 create mode 100644 youtube_dl/extractor/thisvid.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 402e542ae96..3b192e129a9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1249,6 +1249,7 @@
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
+from .thisvid import ThisVidIE
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,
diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
new file mode 100644
index 00000000000..8393c881360
--- /dev/null
+++ b/youtube_dl/extractor/thisvid.py
@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+
+
+class ThisVidIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/videos/big-man-walking26/',
+        'md5': '74a11cac6a738f28c1a37945145a3f96',
+        'info_dict': {
+            'id': '2123318',
+            'ext': 'mp4',
+            'title': 'Big Man Walking26',
+            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2123000/2123318/preview.mp4.jpg',
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://thisvid.com/embed/2123318/',
+        'md5': '74a11cac6a738f28c1a37945145a3f96',
+        'info_dict': {
+            'id': '2123318',
+            'ext': 'mp4',
+            'title': 'Big Man Walking26',
+            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2123000/2123318/preview.mp4.jpg',
+            'age_limit': 18,
+        }
+    }]
+
+    def _real_extract(self, url):
+        main_id = self._match_id(url)
+        webpage = self._download_webpage(url, main_id)
+
+        # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1 and may change in the future.
+        kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
+        if not kvs_version.startswith("5."):
+            self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")
+
+        title = self._html_search_regex(r'<title>(?:Video: )?(.+?)(?: - ThisVid(?:.com| tube))?</title>', webpage, 'title')
+        # video_id, video_url and license_code from the 'flashvars' JSON object:
+        video_id = self._html_search_regex(r"video_id: '([0-9]+)',", webpage, 'video_id')
+        video_url = self._html_search_regex(r"video_url: '(function/0/.+?)',", webpage, 'video_url')
+        license_code = self._html_search_regex(r"license_code: '([0-9$]{16})',", webpage, 'license_code')
+        thumbnail = self._html_search_regex(r"preview_url: '((?:https?:)?//media.thisvid.com/.+?.jpg)',", webpage, 'thumbnail', fatal=False)
+        if thumbnail.startswith("//"):
+            thumbnail = "https:" + thumbnail
+        if (re.match(self._VALID_URL, url).group('type') == "videos"):
+            display_id = main_id
+        else:
+            display_id = self._search_regex(r'<link rel="canonical" href="' + self._VALID_URL + r'">', webpage, 'display_id', fatal=False),
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'url': getrealurl(video_url, license_code),
+            'thumbnail': thumbnail,
+            'age_limit': 18,
+        }
+
+
+def getrealurl(video_url, license_code):
+    urlparts = video_url.split('/')[2:]
+    license = getlicensetoken(license_code)
+    newmagic = urlparts[5][:32]
+
+    for o in range(len(newmagic) - 1, -1, -1):
+        new = ""
+        l = (o + sum([int(n) for n in license[o:]])) % 32
+
+        for i in range(0, len(newmagic)):
+            if i == o:
+                new += newmagic[l]
+            elif i == l:
+                new += newmagic[o]
+            else:
+                new += newmagic[i]
+        newmagic = new
+
+    urlparts[5] = newmagic + urlparts[5][32:]
+    return "/".join(urlparts)
+
+
+def getlicensetoken(license):
+    modlicense = license.replace("$", "").replace("0", "1")
+    center = int(len(modlicense) / 2)
+    fronthalf = int(modlicense[:center + 1])
+    backhalf = int(modlicense[center:])
+
+    modlicense = str(4 * abs(fronthalf - backhalf))
+    retval = ""
+    for o in range(0, center + 1):
+        for i in range(1, 5):
+            retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
+    return retval
\ No newline at end of file

From 664efb3e0693a473669f253d2ae9ba19a38df873 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 1 Jun 2021 19:42:32 +0200
Subject: [PATCH 010/394] Add files via upload

---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 8393c881360..1f00bc6060e 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -33,7 +33,7 @@ def _real_extract(self, url):
         main_id = self._match_id(url)
         webpage = self._download_webpage(url, main_id)
 
-        # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1 and may change in the future.
+        # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1
         kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
         if not kvs_version.startswith("5."):
             self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")

From bed7ee66cfe01ce3fbe3ac49b1e3ff76a55c3829 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 1 Jun 2021 19:45:08 +0200
Subject: [PATCH 011/394] Add files via upload

---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 1f00bc6060e..ae74b03f401 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -94,4 +94,4 @@ def getlicensetoken(license):
     for o in range(0, center + 1):
         for i in range(1, 5):
             retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
-    return retval
\ No newline at end of file
+    return retval

From b5335bda1912982c4ec8c7d6ad94895b59bf1a82 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sat, 5 Jun 2021 19:44:34 +0200
Subject: [PATCH 012/394] Add files via upload

---
 ci.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 ci.yml

diff --git a/ci.yml b/ci.yml
new file mode 100644
index 00000000000..5890a7d85fa
--- /dev/null
+++ b/ci.yml
@@ -0,0 +1,12 @@
+
+prepare:
+  - pip install flake8
+
+stages:
+  - build
+  - test
+
+build-code-job:
+  stage: build
+  script:
+    - flake 8 ./youtube_dl/extractor/thisvid.py
\ No newline at end of file

From 9df9a15c98555cdf1c8b9170b2d0652c1d6b3a39 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sat, 5 Jun 2021 19:45:24 +0200
Subject: [PATCH 013/394] Delete ci.yml

---
 ci.yml | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 ci.yml

diff --git a/ci.yml b/ci.yml
deleted file mode 100644
index 5890a7d85fa..00000000000
--- a/ci.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-prepare:
-  - pip install flake8
-
-stages:
-  - build
-  - test
-
-build-code-job:
-  stage: build
-  script:
-    - flake 8 ./youtube_dl/extractor/thisvid.py
\ No newline at end of file

From a86e76bbf24f4e62cdc32eff32a51597a11eb7da Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 29 Jul 2021 11:46:51 +0100
Subject: [PATCH 014/394] Support alphabetic characters in 7-char NhkVod IDs

See issue #29670
---
 youtube_dl/extractor/nhk.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 8a9331a79f2..c3b44124da7 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -23,7 +24,7 @@ def _call_api(self, m_id, lang, is_video, is_episode, is_clip):
     def _extract_episode_info(self, url, episode=None):
         fetch_episode = episode is None
         lang, m_type, episode_id = re.match(NhkVodIE._VALID_URL, url).groups()
-        if episode_id.isdigit():
+        if len(episode_id) == 7:
             episode_id = episode_id[:4] + '-' + episode_id[4:]
 
         is_video = m_type == 'video'
@@ -84,7 +85,8 @@ def get_clean_field(key):
 
 
 class NhkVodIE(NhkBaseIE):
-    _VALID_URL = r'%s%s(?P<id>\d{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    # the 7-character IDs can have alphabetic chars too: assume [a-z] rather than just [a-f], eg
+    _VALID_URL = r'%s%s(?P<id>[0-9a-z]{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
@@ -124,6 +126,18 @@ class NhkVodIE(NhkBaseIE):
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/j_art-20150903-1/',
         'only_matching': True,
+    }, {
+        # video, alphabetic character in ID #29670
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
+        'info_dict': {
+            'id': 'qfjay6cg',
+            'ext': 'mp4',
+            'title': 'DESIGN TALKS plus - Fishermen’s Finery',
+            'description': 'md5:8a8f958aaafb0d7cb59d38de53f1e448',
+            'thumbnail': r're:^https?:/(/[a-z0-9.-]+)+\.jpg\?w=1920&h=1080$',
+            'upload_date': '20210615',
+            'timestamp': 1623722008,
+        }
     }]
 
     def _real_extract(self, url):

From 6616496875d0863275f47dda18f536c1ef06dfd1 Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Sat, 31 Jul 2021 18:02:53 +0400
Subject: [PATCH 015/394] Implemented referrer request as per:
 https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer

Default policy set to strict-origin-when-cross-origin if no policy is set
---
 test/test_utils.py  | 47 ++++++++++-----------
 youtube_dl/utils.py | 99 ++++++++++++++++++++++++---------------------
 2 files changed, 76 insertions(+), 70 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 26743a813b4..3c5f2eb4871 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -310,36 +310,33 @@ def test_unescape_html(self):
     def test_get_referrer_url(self):
         # No-Referrer
         self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "no-referrer"), None)
-        # No-Referrer-When-Downgrade
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "no-referrer-when-downgrade"), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "no-referrer-when-downgrade"), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "no-referrer-when-downgrade"), None)
         # Origin
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "origin"), "https://example.com/")
-        # Origin-When-Cross-Origin
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "origin-when-cross-origin"), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://mozilla.org", "origin-when-cross-origin"), "https://example.com/")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "origin-when-cross-origin"), "https://example.com/")
-        # Same-Origin
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "same-origin"), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "same-origin"), None)
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "same-origin"), None)
-        # Strict-Origin
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin"), "https://example.com/")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "strict-origin"), None)
-        self.assertEqual(get_referrer_url("http://example.com/page", "http://example.com", "strict-origin"), "http://example.com/")
-        # Strict-Origin-When-Cross-Origin
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "strict-origin-when-cross-origin"), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin-when-cross-origin"), "https://example.com/")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/otherpage", "strict-origin-when-cross-origin"), None)
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "origin"), "https://example.com")
         # Unsafe-Url
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://mozilla.org", "unsafe-url"), "https://example.com/page?q=123")
         self.assertEqual(get_referrer_url("https://example.com/page?q=123", "https://example.com/page?q=123", "unsafe-url"), "https://example.com/page?q=123")
-        # None
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", None), "https://example.com/page")
-        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", None), "https://example.com/")
-        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/otherpage", None), None)
+        # Strict-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin"), "https://example.com")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.org", "strict-origin"), None)
+        self.assertEqual(get_referrer_url("http://example.com/page", "http://example.com", "strict-origin"), "http://example.com")
+        # Strict-Origin-When-Cross-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "strict-origin-when-cross-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "strict-origin-when-cross-origin"), "https://example.com")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/otherpage", "strict-origin-when-cross-origin"), None)
+        # Same-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "same-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "same-origin"), None)
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "same-origin"), None)
+        # Origin-When-Cross-Origin
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "origin-when-cross-origin"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://mozilla.org", "origin-when-cross-origin"), "https://example.com")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com/page", "origin-when-cross-origin"), "https://example.com")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://mozilla.org", "origin-when-cross-origin"), "https://example.com")
+        # No-Referrer-When-Downgrade
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://example.com/otherpage", "no-referrer-when-downgrade"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "https://mozilla.org", "no-referrer-when-downgrade"), "https://example.com/page")
+        self.assertEqual(get_referrer_url("https://example.com/page", "http://example.com", "no-referrer-when-downgrade"), None)
 
     def test_date_from_str(self):
         self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bb4c94c1658..929dedfadd6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2326,57 +2326,66 @@ def bug_reports_message():
     return msg
 
 
-def get_referrer_url(referrer_source, request_url, referrer_policy):
+def get_referrer_url(referrer_source, request_url, policy="strict-origin-when-cross-origin"):
     # Returns correct referrer url based on the site policy
+    # Resources used:
+    # https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#examples
-    parsed_referrer_source = compat_urllib_parse_urlparse(referrer_source)
-    parsed_request_url = compat_urllib_parse_urlparse(request_url)
-    if referrer_policy == "no-referrer":
+    # https://github.com/scrapy/scrapy/blob/master/scrapy/spidermiddlewares/referer.py
+
+    # Strip URL as per https://w3c.github.io/webappsec-referrer-policy/#strip-url
+    def strip_url(url, origin_only=False):
+        if url is None:
+            return "no-referrer"
+        parsed_url = compat_urllib_parse_urlparse(url)
+        if parsed_url.username:
+            parsed_url = parsed_url._replace(username=None)
+        if parsed_url.password:
+            parsed_url = parsed_url._replace(password=None)
+        if parsed_url.fragment:
+            parsed_url = parsed_url._replace(fragment='')
+        if origin_only:
+            parsed_url = parsed_url._replace(path='')
+            parsed_url = parsed_url._replace(query='')
+        return parsed_url.geturl()
+
+    # https://w3c.github.io/webappsec-secure-contexts/#is-origin-trustworthy
+    # More checks to determine the trustworthiness of a URL, the URL scheme is one check
+    def is_tls(url):
+        return compat_urllib_parse_urlparse(url).scheme in ('https', 'ftps')
+    referrer_url = strip_url(referrer_source)
+    referrer_origin = strip_url(referrer_source, origin_only=True)
+    if len(referrer_url) > 4096:
+        referrer_url = referrer_origin
+    if policy == "no-referrer":
         return None
-    elif referrer_policy == "no-referrer-when-downgrade":
-        if "http" == parsed_request_url.scheme and "https" == parsed_referrer_source.scheme:
+    elif policy == "origin":
+        return referrer_origin
+    elif policy == "unsafe-url":
+        return referrer_url
+    elif policy == "strict-origin":
+        if is_tls(referrer_url) and not is_tls(request_url):
             return None
-        elif "https" == parsed_request_url.scheme:
-            return referrer_source
-    elif referrer_policy == "origin":
-        return compat_urlparse.urljoin(referrer_source, "/")
-    elif referrer_policy == "origin-when-cross-origin":
-        if parsed_referrer_source.netloc == parsed_request_url.netloc:
-            if parsed_referrer_source.scheme == parsed_request_url.scheme:
-                return referrer_source
-            elif "http" == parsed_request_url.scheme:
-                return compat_urlparse.urljoin(referrer_source, "/")
-        else:
-            return compat_urlparse.urljoin(referrer_source, "/")
-    elif referrer_policy == "same-origin":
-        if parsed_referrer_source.netloc == parsed_request_url.netloc:
-            if parsed_referrer_source.scheme == parsed_request_url.scheme:
-                return referrer_source
-            else:
-                return None
         else:
+            return referrer_origin
+    elif policy == "strict-origin-when-cross-origin":
+        if referrer_origin == strip_url(request_url, True):
+            return referrer_url
+        elif is_tls(referrer_url) and not is_tls(request_url):
             return None
-    elif referrer_policy == "strict-origin":
-        if "http" == parsed_referrer_source.scheme:
-            return compat_urlparse.urljoin(referrer_source, "/")
-        elif parsed_referrer_source.netloc == parsed_request_url.netloc:
-            if parsed_referrer_source.scheme == parsed_request_url.scheme:
-                return referrer_source
-            else:
-                return None
-        else:
-            return compat_urlparse.urljoin(referrer_source, "/")
-    elif referrer_policy == "strict-origin-when-cross-origin" or referrer_policy is None:
-        # This is the default fallback if no referrer policy is defined
-        if parsed_referrer_source.netloc == parsed_request_url.netloc:
-            if parsed_referrer_source.scheme == parsed_request_url.scheme:
-                return referrer_source
-            else:
-                return None
-        else:
-            return compat_urlparse.urljoin(referrer_source, "/")
-    elif referrer_policy == "unsafe-url":
-        return referrer_source
+        return referrer_origin
+    elif policy == "same-origin":
+        if referrer_origin == strip_url(request_url, True):
+            return referrer_url
+        return None
+    elif policy == "origin-when-cross-origin":
+        if referrer_origin == strip_url(request_url, True):
+            return referrer_url
+        return referrer_origin
+    elif policy == "no-referrer-when-downgrade":
+        if is_tls(referrer_url) and not is_tls(request_url):
+            return None
+        return referrer_url
 
 
 class YoutubeDLError(Exception):

From da49d740fe6ffcffbfd1d693f045653155cbc863 Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Thu, 12 Aug 2021 23:50:46 +0400
Subject: [PATCH 016/394] Updated so strict-origin-when-cross-origin is set by
 default

---
 youtube_dl/extractor/common.py | 3 ++-
 youtube_dl/utils.py            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 942d61fc402..4a2bf5f8140 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2496,7 +2496,8 @@ def parse_content_type(content_type):
             return {}
 
         def get_referrer_policy_from_meta_element(page):
-            return self._html_search_meta('referrer', page)
+            policy = self._html_search_meta('referrer', page)
+            return policy if policy is not None else "strict-origin-when-cross-origin"
 
         def _media_formats(src, cur_media_type, type_info={}):
             full_url = absolute_url(src)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0fa93f28cac..65b4afb0c89 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2327,7 +2327,7 @@ def bug_reports_message():
     return msg
 
 
-def get_referrer_url(referrer_source, request_url, policy="strict-origin-when-cross-origin"):
+def get_referrer_url(referrer_source, request_url, policy):
     # Returns correct referrer url based on the site policy
     # Resources used:
     # https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer

From aaae1ed1a29a44db932dd351fa4ae903b543524b Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Sat, 14 Aug 2021 12:12:02 +0400
Subject: [PATCH 017/394] Added support for streamye

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamye.py   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/streamye.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..5232cff9707 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1176,6 +1176,7 @@
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streamye import StreamYEIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamye.py b/youtube_dl/extractor/streamye.py
new file mode 100644
index 00000000000..eb32bd42933
--- /dev/null
+++ b/youtube_dl/extractor/streamye.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamYEIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?streamye\.com/(?P<id>[a-zA-Z0-9]+)'
+    _TEST = {
+        'url': 'https://streamye.com/vYbcQ2H',
+        'info_dict': {
+            'id': '7bb3fb92faca2ff47a64964637cda2fd',
+            'ext': 'm4a',
+            'title': '7bb3fb92faca2ff47a64964637cda2fd',
+        },
+        'params': {
+            'format': 'bestvideo,bestaudio',
+            'skip_download': True
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._html_search_regex(
+            r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1',
+            webpage, 'url', group="url"
+        )
+
+        return self.url_result(video_url, ie="Generic")

From d1adf8ccb86d97fd45cb52b970f9744b8a1c5629 Mon Sep 17 00:00:00 2001
From: LewdyCoder <88900506+LewdyCoder@users.noreply.github.com>
Date: Sun, 15 Aug 2021 16:55:52 +0200
Subject: [PATCH 018/394] Added little explaination in description

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 58ab3a4b894..bbdf6949007 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, on the condition the class ends with IE.
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):

From 10832d0da4e0c22a1a5d1869c1ec2aa9b50235df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81lvaro=20Mond=C3=A9jar=20Rubio?=
 <mondejar1994@gmail.com>
Date: Wed, 18 Aug 2021 13:37:32 +0200
Subject: [PATCH 019/394] [rtve:alacarta] Add support for 'play' path subpart
 in URL

---
 youtube_dl/extractor/rtve.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index d2fb754cf57..b6f3be689f5 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -27,7 +27,7 @@
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?((alacarta|play)/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -40,6 +40,17 @@ class RTVEALaCartaIE(InfoExtractor):
             'series': 'Balonmano',
         },
         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
+    }, {
+        'url': 'http://www.rtve.es/play/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
+        'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
+        'info_dict': {
+            'id': '2491869',
+            'ext': 'mp4',
+            'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
+            'duration': 5024.566,
+            'series': 'Balonmano',
+        },
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
     }, {
         'note': 'Live stream',
         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',

From 8f6a09b9213808a5698ec52d025ac955ba16cab6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81lvaro=20Mond=C3=A9jar=20Rubio?=
 <mondejar1994@gmail.com>
Date: Wed, 18 Aug 2021 14:58:55 +0200
Subject: [PATCH 020/394] Add support for 'playz' path subpart

---
 youtube_dl/extractor/rtve.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index b6f3be689f5..f060ffdc9e1 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -27,7 +27,7 @@
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?((alacarta|play)/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?((alacarta|playz?)/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -51,6 +51,17 @@ class RTVEALaCartaIE(InfoExtractor):
             'series': 'Balonmano',
         },
         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
+    }, {
+        'url': 'http://www.rtve.es/playz/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
+        'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
+        'info_dict': {
+            'id': '2491869',
+            'ext': 'mp4',
+            'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
+            'duration': 5024.566,
+            'series': 'Balonmano',
+        },
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
     }, {
         'note': 'Live stream',
         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',

From 197215782bb5df5d1632deb7f275ab03c1468a60 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 22 Aug 2021 02:57:02 +0100
Subject: [PATCH 021/394] Small fixes to utils and compat and test

---
 test/test_compat.py  |  6 +++++-
 test/test_utils.py   |  4 ++--
 youtube_dl/compat.py |  9 ++++++---
 youtube_dl/utils.py  | 10 ++++++----
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 86ff389fdfc..04581e6ca2b 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -39,8 +39,12 @@ def test_compat_setenv(self):
         self.assertEqual(compat_getenv(test_var), test_str)
 
     def test_compat_expanduser(self):
+        from youtube_dl.compat import compat_os_name
         old_home = os.environ.get('HOME')
-        test_str = r'C:\Documents and Settings\тест\Application Data'
+        if compat_os_name in ('nt', 'ce'):
+            test_str = r'C:\Documents and Settings\тест\Application Data'
+        else:
+            test_str = '/home/тест'
         compat_setenv('HOME', test_str)
         self.assertEqual(compat_expanduser('~'), test_str)
         compat_setenv('HOME', old_home or '')
diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e1e..6f894579277 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -480,7 +480,7 @@ def test_shell_quote(self):
         args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
         self.assertEqual(
             shell_quote(args),
-            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
+            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if not(compat_os_name in ('nt', 'ce')) else '''ffmpeg -i "ñ€ß'.mp4"''')
 
     def test_float_or_none(self):
         self.assertEqual(float_or_none('42.42'), 42.42)
@@ -1085,7 +1085,7 @@ def test_intlist_to_bytes(self):
     def test_args_to_str(self):
         self.assertEqual(
             args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
-            'foo ba/r -baz \'2 be\' \'\'' if compat_os_name != 'nt' else 'foo ba/r -baz "2 be" ""'
+            'foo ba/r -baz \'2 be\' \'\'' if not(compat_os_name in ('nt', 'ce')) else 'foo ba/r -baz "2 be" ""'
         )
 
     def test_parse_filesize(self):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..90dd2045774 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2700,16 +2700,19 @@ def compat_setenv(key, value, env=os.environ):
     # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
 
     def compat_getenv(key, default=None):
-        from .utils import get_filesystem_encoding
         env = os.getenv(key, default)
         if env:
-            env = env.decode(get_filesystem_encoding())
+            from .utils import get_filesystem_encoding
+            encoding = get_filesystem_encoding()
+            env = env.decode(encoding)
+            if not encoding.lower().startswith('ut'):
+                env = env.encode('utf-8').decode('unicode-escape')
         return env
 
     def compat_setenv(key, value, env=os.environ):
         def encode(v):
             from .utils import get_filesystem_encoding
-            return v.encode(get_filesystem_encoding()) if isinstance(v, compat_str) else v
+            return v.encode(get_filesystem_encoding(), 'backslashreplace') if isinstance(v, compat_str) else v
         env[encode(key)] = encode(value)
 
     # HACK: The default implementations of os.path.expanduser from cpython do not decode
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58de..eaf86bb441d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2245,7 +2245,8 @@ def encodeFilename(s, for_subprocess=False):
     if sys.platform.startswith('java'):
         return s
 
-    return s.encode(get_subprocess_encoding(), 'ignore')
+    # If encoding is (eg) 'ascii', use escape sequences (allows round-trip test)
+    return s.encode(get_subprocess_encoding(), 'backslashreplace')
 
 
 def decodeFilename(b, for_subprocess=False):
@@ -3354,8 +3355,7 @@ def read(self, *args):
 
 
 def get_filesystem_encoding():
-    encoding = sys.getfilesystemencoding()
-    return encoding if encoding is not None else 'utf-8'
+    return sys.getfilesystemencoding() or sys.getdefaultencoding() or 'utf-8'
 
 
 def shell_quote(args):
@@ -3365,6 +3365,8 @@ def shell_quote(args):
         if isinstance(a, bytes):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
+            if not encoding.lower().startswith('ut'):
+                a = a.encode('utf-8').decode('unicode-escape')
         quoted_args.append(compat_shlex_quote(a))
     return ' '.join(quoted_args)
 
@@ -4610,7 +4612,7 @@ def parse_node(node):
             continue
         default_style.update(style)
 
-    for para, index in zip(paras, itertools.count(1)):
+    for index, para in enumerate(paras, 1):
         begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
         dur = parse_dfxp_time_expr(para.attrib.get('dur'))

From 1e222005ba6bb9c288dd3a2b777ca5664b220c81 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 29 Aug 2021 05:34:20 +0100
Subject: [PATCH 022/394] Fix urlhandle_detect_ext() non-ASCII error in Py2,
 with test

---
 test/test_utils.py  | 25 +++++++++++++++++++++++++
 youtube_dl/utils.py |  8 ++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 6f894579277..44a4f6ff7ef 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -105,6 +105,7 @@
     cli_valueless_option,
     cli_bool_option,
     parse_codecs,
+    urlhandle_detect_ext,
 )
 from youtube_dl.compat import (
     compat_chr,
@@ -1475,6 +1476,30 @@ def test_clean_podcast_url(self):
         self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3')
         self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3')
 
+    def test_urlhandle_detect_ext(self):
+
+        class UrlHandle(object):
+            _info = {}
+
+            def __init__(self, info):
+                self._info = info
+
+            @property
+            def headers(self):
+                return self._info
+
+        # header with non-ASCII character and contradictory Content-Type
+        urlh = UrlHandle({
+            'Content-Disposition': b'attachment; filename="Epis\xf3dio contains non-ASCI ISO 8859-1 character.mp3"',
+            'Content-Type': b'audio/aac',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with no Content-Disposition
+        urlh = UrlHandle({
+            'Content-Type': b'audio/mp3',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index eaf86bb441d..5dde9768daa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4288,7 +4288,10 @@ def parse_codecs(codecs_str):
 def urlhandle_detect_ext(url_handle):
     getheader = url_handle.headers.get
 
-    cd = getheader('Content-Disposition')
+    def encode_compat_str_or_none(x, encoding='iso-8859-1', errors='ignore'):
+        return encode_compat_str(x, encoding=encoding, errors=errors) if x else None
+
+    cd = encode_compat_str_or_none(getheader('Content-Disposition'))
     if cd:
         m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
         if m:
@@ -4296,7 +4299,8 @@ def urlhandle_detect_ext(url_handle):
             if e:
                 return e
 
-    return mimetype2ext(getheader('Content-Type'))
+    ct = encode_compat_str_or_none(getheader('Content-Type'))
+    return mimetype2ext(ct)
 
 
 def encode_data_uri(data, mime_type):

From 38bcac9ae42e4524d7e15a3422d36dd127dcdb66 Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Sat, 11 Sep 2021 00:39:17 +0400
Subject: [PATCH 023/394] Removed StreamYE

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/streamye.py   | 30 ------------------------------
 2 files changed, 31 deletions(-)
 delete mode 100644 youtube_dl/extractor/streamye.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5232cff9707..6e8fc3961b7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1176,7 +1176,6 @@
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
-from .streamye import StreamYEIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamye.py b/youtube_dl/extractor/streamye.py
deleted file mode 100644
index eb32bd42933..00000000000
--- a/youtube_dl/extractor/streamye.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class StreamYEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?streamye\.com/(?P<id>[a-zA-Z0-9]+)'
-    _TEST = {
-        'url': 'https://streamye.com/vYbcQ2H',
-        'info_dict': {
-            'id': '7bb3fb92faca2ff47a64964637cda2fd',
-            'ext': 'm4a',
-            'title': '7bb3fb92faca2ff47a64964637cda2fd',
-        },
-        'params': {
-            'format': 'bestvideo,bestaudio',
-            'skip_download': True
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._html_search_regex(
-            r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1',
-            webpage, 'url', group="url"
-        )
-
-        return self.url_result(video_url, ie="Generic")

From f798b40cf332c0a00f167fcfc9d560fa8f795c13 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 13 Sep 2021 01:00:04 +0100
Subject: [PATCH 024/394] Disambiguate 4-digit year and time-zone suffix

Restore check omitted from extract_timezone(); adjust DATE_FORMATS_DAY/MONTH_FIRST; add tests.
---
 test/test_utils.py  |  6 ++++++
 youtube_dl/utils.py | 24 ++++++++++++++++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 44a4f6ff7ef..14607f6b8cb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -371,6 +371,12 @@ def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
         self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
         self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
+        self.assertEqual(unified_timestamp('11:31 17-Jun-2021'), 1623929460)
+        self.assertEqual(unified_timestamp('11:31 17-Jun-2021-0000'), 1623929460)
+        from youtube_dl.utils import DATE_FORMATS_DAY_FIRST
+        DATE_FORMATS_DAY_FIRST.append('%H:%M %d-%m-%Y')
+        self.assertEqual(unified_timestamp('17:30 27-02-2016'), 1456594200)
+        self.assertEqual(unified_timestamp('17:30 27-02-2016-0000'), 1456594200)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5dde9768daa..90eb9f93c44 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # coding: utf-8
 
 from __future__ import unicode_literals
@@ -1717,8 +1716,6 @@ def random_user_agent():
                                         'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
 
 DATE_FORMATS = (
-    '%d %B %Y',
-    '%d %b %Y',
     '%B %d %Y',
     '%B %dst %Y',
     '%B %dnd %Y',
@@ -1763,6 +1760,11 @@ def random_user_agent():
     '%d/%m/%Y',
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
+    '%d %B %Y',
+    '%d %b %Y',
+    '%d-%b-%Y',
+    '%H:%M %d-%b-%Y',
+    '%H:%M:%S %d-%b-%Y',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
@@ -1772,6 +1774,11 @@ def random_user_agent():
     '%m/%d/%Y',
     '%m/%d/%y',
     '%m/%d/%Y %H:%M:%S',
+    '%B %d %Y',
+    '%b %d %Y',
+    '%b-%d-%Y',
+    '%H:%M %b-%d-%Y',
+    '%H:%M:%S %b-%d-%Y',
 ])
 
 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
@@ -2939,7 +2946,16 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
 
 def extract_timezone(date_str):
     m = re.search(
-        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+        r'''(?x)
+            ^.{8,}?                                              # >=8 char non-TZ prefix, if present
+            (?P<tz>Z|                                            # just the UTC Z, or
+                (?:(?<=.\b\d{4}|\b\d{2}:\d\d)|                   # preceded by 4 digits or hh:mm or
+                   (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d))     # not preceded by 3 alpha word or >= 4 alpha or 2 digits
+                   [ ]?                                          # optional space
+                (?P<sign>\+|-)                                   # +/-
+                (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})       # hh[:]mm
+            $)
+        ''',
         date_str)
     if not m:
         timezone = datetime.timedelta()

From 041cd20a71ed11b5aa785a0239c01ccf67d092f2 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Wed, 25 Aug 2021 18:56:01 +0100
Subject: [PATCH 025/394] Update Beeg extractor for new API

---
 youtube_dl/extractor/beeg.py | 83 ++++++++++++++++++++++++++++++++----
 1 file changed, 74 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 5788d13baae..551b789e557 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -6,13 +6,17 @@
     compat_urlparse,
 )
 from ..utils import (
+    dict_get,
     int_or_none,
+    parse_iso8601,
+    try_get,
     unified_timestamp,
+    urljoin,
 )
 
 
 class BeegIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?beeg\.(?:com|porn(?:/video)?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?beeg\.(?:com|porn(?:/video)?)/-?(?P<id>\d+)'
     _TESTS = [{
         # api/v6 v1
         'url': 'http://beeg.com/5416503',
@@ -27,7 +31,8 @@ class BeegIE(InfoExtractor):
             'duration': 383,
             'tags': list,
             'age_limit': 18,
-        }
+        },
+        'skip': 'no longer available',
     }, {
         # api/v6 v2
         'url': 'https://beeg.com/1941093077?t=911-1391',
@@ -42,13 +47,23 @@ class BeegIE(InfoExtractor):
     }, {
         'url': 'https://beeg.porn/5416503',
         'only_matching': True,
+    }, {
+        'url': 'https://beeg.com/-01882333214',
+        'md5': 'f8fd97a58a09bc6d2ac25b6bbc98c220',
+        'info_dict': {
+            'id': '01882333214',
+            'ext': 'mp4',
+            'title': 'Be mine',
+            'description': 'Lil cam dork living in Canada with 4 kitties',
+            'timestamp': 1628218803,
+            'upload_date': '20210806',
+            'duration': 559,
+            'tags': list,
+            'age_limit': 18,
+        },
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
+    def _old_real_extract(self, url, video_id, webpage):
         beeg_version = self._search_regex(
             r'beeg_version\s*=\s*([\da-zA-Z_-]+)', webpage, 'beeg version',
             default='1546225636701')
@@ -67,11 +82,14 @@ def _real_extract(self, url):
         else:
             query = {'v': 1}
 
-        for api_path in ('', 'api.'):
+        def reverse_enumerate(l):
+            return zip(reversed(zip(*enumerate(l))[0]), l)
+
+        for rcnt, api_path in reverse_enumerate(('', 'api.')):
             video = self._download_json(
                 'https://%sbeeg.com/api/v6/%s/video/%s'
                 % (api_path, beeg_version, video_id), video_id,
-                fatal=api_path == 'api.', query=query)
+                fatal=(rcnt == 0), query=query)
             if video:
                 break
 
@@ -114,3 +132,50 @@ def _real_extract(self, url):
             'formats': formats,
             'age_limit': self._rta_search(webpage),
         }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        # leading 0s stripped from ID by site, though full ID also works
+        video = self._download_json(
+            'https://store.externulls.com/facts/file/%d' % int(video_id),
+            video_id, fatal=False)
+        if video:
+            video_data = try_get(video, lambda x: x['file'], dict) or {}
+            video_data.update(video_data.get('stuff', {}))
+            video_data.update(try_get(video, lambda x: x['fc_facts'][0], dict))
+            for format_id, key in try_get(video_data, lambda x: iter(x['resources'].items())) or []:
+                if not format_id:
+                    continue
+                formats.append({
+                    'url': urljoin('https://video.beeg.com', key),
+                    'format_id': format_id,
+                    'height': int_or_none(format_id.rsplit('_', 1)[-1]),
+                })
+            if formats:
+                self._sort_formats(formats)
+                title = video_data.get('sf_name') or 'Beeg video %s' % video_id
+                tags = try_get(video_data, lambda v: [tag['tg_slug'] for tag in v['tags'] if tag.get('tg_slug')])
+                result = {
+                    'id': video_id,
+                    'title': video_data.get('sf_name') or 'Beeg video %s' % video_id,
+                    'description': video_data.get('sf_story', title),
+                    'timestamp': parse_iso8601(video_data.get('fc_created')),
+                    'duration': int_or_none(dict_get(video_data, ('fl_duration', 'sf_duration'))),
+                    'tags': tags,
+                    'formats': formats,
+                    'age_limit': self._rta_search(webpage),
+                    'height': int_or_none(video_data.get('fl_height')),
+                    'width': int_or_none(video_data.get('fl_width')),
+                }
+                set_id = video_data['resources'].get('set_id')
+                thumb_id = try_get(video_data, lambda x: x['fc_facts'][0]['fc_thumbs'][0], int)
+                if set_id and thumb_id:
+                    result['thumbnail'] = (
+                        'https://thumbs-015.externulls.com/sets/%s/thumbs/%s-%0.4d.jpg' % (set_id, set_id, thumb_id))
+                return result
+
+        return self._old_real_extract(url, video_id, webpage)

From 74b53109ff5923df82498ee967c8c263c18f9721 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 10 Oct 2021 12:42:51 +0100
Subject: [PATCH 026/394] Detect extension from any RFC Content-Disposition
 syntax

Add support for unquoted token and RFC 5987 extended parameter syntax
---
 test/test_utils.py  | 27 +++++++++++++++++++++++++++
 youtube_dl/utils.py | 20 +++++++++++++++++---
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 14607f6b8cb..50fb5f10147 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1505,6 +1505,33 @@ def headers(self):
             'Content-Type': b'audio/mp3',
         })
         self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with Content-Disposition and unquoted filename
+        urlh = UrlHandle({
+            'Content-Disposition': b'attachment; filename=unquoted_filename_token.mp3',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with Content-Disposition including spacing and uppercase
+        urlh = UrlHandle({
+            'Content-Disposition': b'ATTACHMENT; FileName = unquoted_filename_token.mp3',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with Content-Disposition and extended filename parameter syntax
+        urlh = UrlHandle({
+            'Content-Disposition': b"attachment; filename*=iso8859-15''costs%201%A4%20filename.mp3",
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with Content-Disposition and both filename parameter syntaxes
+        urlh = UrlHandle({
+            'Content-Disposition': b'''attachment; filename="should ignore.mp4";
+             FileName* = iso8859-15''costs%201%A4%20filename.mp3''',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
+        # header with Content-Disposition and 'wrong' order of both syntaxes
+        urlh = UrlHandle({
+            'Content-Disposition': b'''attachment; filename*=iso8859-15''costs%201%A4%20filename.mp3;
+            filename="should ignore.mp4"''',
+        })
+        self.assertEqual(urlhandle_detect_ext(urlh), 'mp3')
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 90eb9f93c44..02631406c29 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -58,9 +58,10 @@
     compat_struct_unpack,
     compat_urllib_error,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
-    compat_urllib_parse_unquote_plus,
     compat_urllib_request,
     compat_urlparse,
     compat_xpath,
@@ -4309,9 +4310,22 @@ def encode_compat_str_or_none(x, encoding='iso-8859-1', errors='ignore'):
 
     cd = encode_compat_str_or_none(getheader('Content-Disposition'))
     if cd:
-        m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
+        m = re.match(r'''(?xi)
+            attachment;\s*
+            (?:filename\s*=[^;]+?;\s*)?                    # possible initial filename=...;, ignored
+            filename(?P<x>\*)?\s*=\s*                      # filename/filename* =
+                (?(x)(?P<charset>\S+?)'[\w-]*'|(?P<q>")?)  # if * then charset'...' else maybe "
+                (?P<filename>(?(q)[^"]+(?=")|[^\s;]+))         # actual name of file
+            ''', cd)
         if m:
-            e = determine_ext(m.group('filename'), default_ext=None)
+            m = m.groupdict()
+            filename = m.get('filename')
+            if m.get('x'):
+                try:
+                    filename = compat_urllib_parse_unquote(filename, encoding=m.get('charset', 'utf-8'))
+                except LookupError:  # unrecognised character set name
+                    pass
+            e = determine_ext(filename, default_ext=None)
             if e:
                 return e
 

From a941a68d5f9fa3d785713b73c854190a1b23d345 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 11 Oct 2021 13:10:50 +0100
Subject: [PATCH 027/394] Avoid scrubbing supplied cookie file on failed update

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 02631406c29..fcb78757032 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2781,7 +2781,8 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False):
             if cookie.expires is None:
                 cookie.expires = 0
 
-        with io.open(filename, 'w', encoding='utf-8') as f:
+        with io.open(filename, 'r+', encoding='utf-8') as f:
+            assert 0 == f.seek(0)
             f.write(self._HEADER)
             now = time.time()
             for cookie in self:
@@ -2813,6 +2814,7 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False):
                 f.write(
                     '\t'.join([cookie.domain, initial_dot, cookie.path,
                                secure, expires, name, value]) + '\n')
+                assert f.tell() == f.truncate()
 
     def load(self, filename=None, ignore_discard=False, ignore_expires=False):
         """Load cookies from a file."""

From 3f209889955c90f465b14b8de24b4da86602e732 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 11 Oct 2021 16:58:57 +0100
Subject: [PATCH 028/394] Back-port Paramount+ extractor (etc) from yt-dlp

With supporting changes to CBS and ThePlatform extractors
---
 youtube_dl/extractor/cbs.py           | 131 ++++++++++++++++-------
 youtube_dl/extractor/extractors.py    |   4 +
 youtube_dl/extractor/paramountplus.py | 145 ++++++++++++++++++++++++++
 youtube_dl/extractor/theplatform.py   |  21 +++-
 4 files changed, 259 insertions(+), 42 deletions(-)
 create mode 100644 youtube_dl/extractor/paramountplus.py

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index c79e55a75da..8ff2b2240be 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -8,6 +8,7 @@
     xpath_element,
     xpath_text,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -25,10 +26,62 @@ def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
                     })
         return subtitles
 
+    def _extract_common_video_info(self, content_id, asset_types, mpx_acc, extra_info):
+        tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id)
+        tp_release_url = 'http://link.theplatform.com/s/' + tp_path
+        info = self._extract_theplatform_metadata(tp_path, content_id)
+
+        formats, subtitles = [], {}
+        last_e = None
+        for asset_type, query in asset_types.items():
+            try:
+                tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                    update_url_query(tp_release_url, query), content_id,
+                    'Downloading %s SMIL data' % asset_type)
+            except ExtractorError as e:
+                last_e = e
+                if asset_type != 'fallback':
+                    continue
+                query['formats'] = ''  # blank query to check if expired
+            try:
+                tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                    update_url_query(tp_release_url, query), content_id,
+                    'Downloading %s SMIL data, trying again with another format' % asset_type)
+            except ExtractorError as e:
+                last_e = e
+                continue
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+        if last_e and not formats:
+            raise last_e
+        self._sort_formats(formats)
+
+        extra_info.update({
+            'id': content_id,
+            'formats': formats,
+            'subtitles': subtitles,
+        })
+        info.update({k: v for k, v in extra_info.items() if v is not None})
+        return info
+
+    def _extract_video_info(self, *args, **kwargs):
+        # Extract assets + metadata and call _extract_common_video_info
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _real_extract(self, url):
+        return self._extract_video_info(self._match_id(url))
+
 
 class CBSIE(CBSBaseIE):
-    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:(?:cbs|paramountplus)\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
+    _VALID_URL = r'''(?x)
+        (?:
+            cbs:|
+            https?://(?:www\.)?(?:
+                cbs\.com/(?:shows/[^/]+/video|movies/[^/]+)/|
+                colbertlateshow\.com/(?:video|podcasts)/)
+        )(?P<id>[\w-]+)'''
 
+    # All tests are blocked outside US
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
@@ -46,70 +99,70 @@ class CBSIE(CBSBaseIE):
             'skip_download': True,
         },
         '_skip': 'Blocked outside the US',
+    }, {
+        'url': 'https://www.cbs.com/shows/the-late-show-with-stephen-colbert/video/60icOhMb9NcjbcWnF_gub9XXHdeBcNk2/the-late-show-6-23-21-christine-baranski-joy-oladokun-',
+        'info_dict': {
+            'id': '60icOhMb9NcjbcWnF_gub9XXHdeBcNk2',
+            'title': 'The Late Show - 6/23/21 (Christine Baranski, Joy Oladokun)',
+            'timestamp': 1624507140,
+            'description': 'md5:e01af24e95c74d55e8775aef86117b95',
+            'uploader': 'CBSI-NEW',
+            'upload_date': '20210624',
+        },
+        'params': {
+            'ignore_no_formats_error': True,
+            'skip_download': True,
+        },
+        'expected_warnings': [
+            'This content expired on', 'No video formats found', 'Requested format is not available'],
     }, {
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,
     }, {
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
-    }, {
-        'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/',
-        'only_matching': True,
     }]
 
     def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
         items_data = self._download_xml(
-            'http://can.cbs.com/thunder/player/videoPlayerService.php',
+            'https://can.cbs.com/thunder/player/videoPlayerService.php',
             content_id, query={'partner': site, 'contentId': content_id})
         video_data = xpath_element(items_data, './/item')
-        title = xpath_text(video_data, 'videoTitle', 'title', True)
-        tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id)
-        tp_release_url = 'http://link.theplatform.com/s/' + tp_path
+        title = xpath_text(video_data, 'videoTitle', 'title') or xpath_text(video_data, 'videotitle', 'title')
 
-        asset_types = []
-        subtitles = {}
-        formats = []
-        last_e = None
+        asset_types = {}
+        has_drm = False
         for item in items_data.findall('.//item'):
             asset_type = xpath_text(item, 'assetType')
-            if not asset_type or asset_type in asset_types or 'HLS_FPS' in asset_type or 'DASH_CENC' in asset_type:
-                continue
-            asset_types.append(asset_type)
             query = {
                 'mbr': 'true',
                 'assetTypes': asset_type,
             }
-            if asset_type.startswith('HLS') or asset_type in ('OnceURL', 'StreamPack'):
+            if not asset_type:
+                # fallback for content_ids that videoPlayerService doesn't return anything for
+                asset_type = 'fallback'
+                query['formats'] = 'M3U+none,MPEG4,M3U+appleHlsEncryption,MP3'
+                del query['assetTypes']
+            if asset_type in asset_types:
+                continue
+            elif any(excluded in asset_type for excluded in ('HLS_FPS', 'DASH_CENC', 'OnceURL')):
+                if 'DASH_CENC' in asset_type:
+                    has_drm = True
+                continue
+            if asset_type.startswith('HLS') or 'StreamPack' in asset_type:
                 query['formats'] = 'MPEG4,M3U'
             elif asset_type in ('RTMP', 'WIFI', '3G'):
                 query['formats'] = 'MPEG4,FLV'
-            try:
-                tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                    update_url_query(tp_release_url, query), content_id,
-                    'Downloading %s SMIL data' % asset_type)
-            except ExtractorError as e:
-                last_e = e
-                continue
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-        if last_e and not formats:
-            raise last_e
-        self._sort_formats(formats)
+            asset_types[asset_type] = query
 
-        info = self._extract_theplatform_metadata(tp_path, content_id)
-        info.update({
-            'id': content_id,
+        if not asset_types and has_drm:
+            raise ExtractorError('Only DRM formats found', video_id=content_id, expected=True)
+
+        return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={
             'title': title,
             'series': xpath_text(video_data, 'seriesTitle'),
             'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
             'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
             'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
-            'thumbnail': xpath_text(video_data, 'previewImageURL'),
-            'formats': formats,
-            'subtitles': subtitles,
+            'thumbnail': url_or_none(xpath_text(video_data, 'previewImageURL')),
         })
-        return info
-
-    def _real_extract(self, url):
-        content_id = self._match_id(url)
-        return self._extract_video_info(content_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..da3c7a828da 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -889,6 +889,10 @@
     PalcoMP3VideoIE,
 )
 from .pandoratv import PandoraTVIE
+from .paramountplus import (
+     ParamountPlusIE,
+     ParamountPlusSeriesIE,
+)
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
diff --git a/youtube_dl/extractor/paramountplus.py b/youtube_dl/extractor/paramountplus.py
new file mode 100644
index 00000000000..86c07f61f7b
--- /dev/null
+++ b/youtube_dl/extractor/paramountplus.py
@@ -0,0 +1,145 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .cbs import CBSBaseIE
+from ..utils import (
+    int_or_none,
+    url_or_none,
+)
+
+
+class ParamountPlusIE(CBSBaseIE):
+    _VALID_URL = r'''(?x)
+        (?:
+            paramountplus:|
+            https?://(?:www\.)?(?:
+                paramountplus\.com/(?:shows/[^/]+/video|movies/[^/]+)/
+        )(?P<id>[\w-]+))'''
+
+    # All tests are blocked outside US
+    _TESTS = [{
+        'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/',
+        'info_dict': {
+            'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k',
+            'ext': 'mp4',
+            'title': 'CatDog - Climb Every CatDog/The Canine Mutiny',
+            'description': 'md5:7ac835000645a69933df226940e3c859',
+            'duration': 1426,
+            'timestamp': 920264400,
+            'upload_date': '19990301',
+            'uploader': 'CBSI-NEW',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://www.paramountplus.com/shows/tooning-out-the-news/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/7-23-21-week-in-review-rep-jahana-hayes-howard-fineman-sen-michael-bennet-sheera-frenkel-cecilia-kang-/',
+        'info_dict': {
+            'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd',
+            'ext': 'mp4',
+            'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)',
+            'description': 'md5:f4adcea3e8b106192022e121f1565bae',
+            'duration': 2506,
+            'timestamp': 1627063200,
+            'upload_date': '20210723',
+            'uploader': 'CBSI-NEW',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://www.paramountplus.com/movies/daddys-home/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC',
+        'info_dict': {
+            'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC',
+            'ext': 'mp4',
+            'title': 'Daddy\'s Home',
+            'upload_date': '20151225',
+            'description': 'md5:9a6300c504d5e12000e8707f20c54745',
+            'uploader': 'CBSI-NEW',
+            'timestamp': 1451030400,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+            'format': 'bestvideo',
+        },
+        'expected_warnings': ['Ignoring subtitle tracks'],  # TODO: Investigate this
+    }, {
+        'url': 'https://www.paramountplus.com/movies/sonic-the-hedgehog/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc',
+        'info_dict': {
+            'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc',
+            'ext': 'mp4',
+            'uploader': 'CBSI-NEW',
+            'description': 'md5:bc7b6fea84ba631ef77a9bda9f2ff911',
+            'timestamp': 1577865600,
+            'title': 'Sonic the Hedgehog',
+            'upload_date': '20200101',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+            'format': 'bestvideo',
+        },
+        'expected_warnings': ['Ignoring subtitle tracks'],
+    }, {
+        'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.paramountplus.com/movies/million-dollar-american-princesses-meghan-and-harry/C0LpgNwXYeB8txxycdWdR9TjxpJOsdCq',
+        'only_matching': True,
+    }]
+
+    def _extract_video_info(self, content_id, mpx_acc=2198311517):
+        items_data = self._download_json(
+            'https://www.paramountplus.com/apps-api/v2.0/androidtv/video/cid/%s.json' % content_id,
+            content_id, query={'locale': 'en-us', 'at': 'ABCqWNNSwhIqINWIIAG+DFzcFUvF8/vcN6cNyXFFfNzWAIvXuoVgX+fK4naOC7V8MLI='}, headers=self.geo_verification_headers())
+
+        asset_types = {
+            item.get('assetType'): {
+                'format': 'SMIL',
+                'formats': 'MPEG4,M3U',
+            } for item in items_data['itemList']
+        }
+        item = items_data['itemList'][-1]
+        return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={
+            'title': item.get('title'),
+            'series': item.get('seriesTitle'),
+            'season_number': int_or_none(item.get('seasonNum')),
+            'episode_number': int_or_none(item.get('episodeNum')),
+            'duration': int_or_none(item.get('duration')),
+            'thumbnail': url_or_none(item.get('thumbnail')),
+        })
+
+
+class ParamountPlusSeriesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P<id>[a-zA-Z0-9-_]+)/?(?:[#?]|$)'
+    _TESTS = [{
+        'url': 'https://www.paramountplus.com/shows/drake-josh',
+        'playlist_mincount': 45,
+        'info_dict': {
+            'id': 'drake-josh',
+        }
+    }, {
+        'url': 'https://www.paramountplus.com/shows/hawaii_five_0/',
+        'playlist_mincount': 240,
+        'info_dict': {
+            'id': 'hawaii_five_0',
+        }
+    }, {
+        'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/',
+        'playlist_mincount': 248,
+        'info_dict': {
+            'id': 'spongebob-squarepants',
+        }
+    }]
+    _API_URL = 'https://www.paramountplus.com/shows/{}/xhr/episodes/page/0/size/100000/xs/0/season/0/'
+
+    def _entries(self, show_name):
+        show_json = self._download_json(self._API_URL.format(show_name), video_id=show_name)
+        if show_json.get('success'):
+            for episode in show_json['result']['data']:
+                yield self.url_result(
+                    'https://www.paramountplus.com%s' % episode['url'],
+                    ie=ParamountPlusIE.ie_key(), video_id=episode['content_id'])
+
+    def _real_extract(self, url):
+        show_name = self._match_id(url)
+        return self.playlist_result(self._entries(show_name), playlist_id=show_name)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index adfe11e3142..65d583ce6f2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -20,6 +20,7 @@
     float_or_none,
     int_or_none,
     sanitized_Request,
+    try_get,
     unsmuggle_url,
     update_url_query,
     xpath_with_ns,
@@ -34,6 +35,15 @@
 class ThePlatformBaseIE(OnceIE):
     _TP_TLD = 'com'
 
+    @classmethod
+    def _match_valid_url(cls, url):
+        # This does not use has/getattr intentionally - we want to know whether
+        # we have cached the regexp for *this* class, whereas getattr would also
+        # match the superclass
+        if '_VALID_URL_RE' not in cls.__dict__:
+            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
+        return cls._VALID_URL_RE.match(url)
+
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(
             smil_url, video_id, note=note, query={'format': 'SMIL'},
@@ -238,7 +248,7 @@ def _real_extract(self, url):
             'countries': smuggled_data.get('geo_countries'),
         })
 
-        mobj = re.match(self._VALID_URL, url)
+        mobj = self._match_valid_url(url)
         provider_id = mobj.group('provider_id')
         video_id = mobj.group('id')
 
@@ -338,6 +348,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
             'uploader': 'NBCU-NEWS',
         },
+        'expected_warnings': ('Empty metadata',),
     }, {
         'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byGuid=nn_netcast_180306.Copy.01',
         'only_matching': True,
@@ -345,7 +356,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
 
     def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}, account_id=None):
         real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
-        entry = self._download_json(real_url, video_id)['entries'][0]
+        entry = self._download_json(real_url, video_id)
+        entry = try_get(entry, lambda x: x['entries'][0], dict)
+        if not entry:
+            self.report_warning('Empty metadata', video_id)
+            return None
         main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else entry.get('plmedia$publicUrl')
 
         formats = []
@@ -404,7 +419,7 @@ def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custo
         return ret
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        mobj = self._match_valid_url(url)
 
         video_id = mobj.group('id')
         provider_id = mobj.group('provider_id')

From a2d0cf524370fc4890a8ca688f503568e09a4d80 Mon Sep 17 00:00:00 2001
From: Ahmed <aasuwaidi24@gmail.com>
Date: Thu, 21 Oct 2021 11:14:08 +0400
Subject: [PATCH 029/394] Added streamff

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamff.py   | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 youtube_dl/extractor/streamff.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..7a331a516cb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1176,6 +1176,7 @@
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streamff import StreamFFIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
new file mode 100644
index 00000000000..83ab398fc27
--- /dev/null
+++ b/youtube_dl/extractor/streamff.py
@@ -0,0 +1,26 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamFFIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?streamff\.com/v/(?P<id>[0-9a-zA-Z]+)'
+    _TEST = {
+        'url': 'https://streamff.com/v/bc22b8',
+        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'info_dict': {
+            'id': '42',
+            'ext': 'mp4',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_url = "https://streamff.streamff.com/uploads/%s.mp4" % video_id
+        return self.url_result(video_url, ie="Generic")

From 15f9c35ab8c7182359d2dc5ac4fdc8e4adfb2e88 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:55:38 +0100
Subject: [PATCH 030/394] Fix test_youtube_playlist_noplaylist

---
 test/test_youtube_lists.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index cf2fdf14f62..72820972e2d 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 # Allow direct execution
@@ -9,11 +10,10 @@
 
 from test.helper import FakeYDL
 
-
 from youtube_dl.extractor import (
+    YoutubeIE,
     YoutubePlaylistIE,
     YoutubeTabIE,
-    YoutubeIE,
 )
 
 
@@ -25,9 +25,11 @@ def assertIsPlaylist(self, info):
     def test_youtube_playlist_noplaylist(self):
         dl = FakeYDL()
         dl.params['noplaylist'] = True
+        dl.params['format'] = 'best'
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
         self.assertEqual(result['_type'], 'url')
+        result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
     def test_youtube_course(self):

From ea31a0bb168c1006c454cc19e222a87871bdd8c8 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:57:10 +0100
Subject: [PATCH 031/394] Remove obsolete test_youtube_course

---
 test/test_youtube_lists.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 72820972e2d..e1636a1a6d6 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -32,16 +32,6 @@ def test_youtube_playlist_noplaylist(self):
         result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
-    def test_youtube_course(self):
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        # TODO find a > 100 (paginating?) videos course
-        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = list(result['entries'])
-        self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
-        self.assertEqual(len(entries), 25)
-        self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
-
     def test_youtube_mix(self):
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)

From 050f74ff0e9cccb30e1eadba0ffef06c11cae5ba Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:03:00 +0100
Subject: [PATCH 032/394] Fix test_youtube_mix

---
 test/test_youtube_lists.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index e1636a1a6d6..fae8a950a4b 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -34,12 +34,14 @@ def test_youtube_playlist_noplaylist(self):
 
     def test_youtube_mix(self):
         dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
-        entries = result['entries']
+        dl.params['format'] = 'best'
+        ie = YoutubeTabIE(dl)
+        result = dl.extract_info('https://www.youtube.com/watch?v=uVJ0Il5WvbE&list=PLhQjrBD2T381k8ul4WQ8SQ165XqY149WW',
+                                 download=False, ie_key=ie.ie_key(), process=True)
+        entries = (result or {}).get('entries', [{'id': 'not_found', }])
         self.assertTrue(len(entries) >= 50)
         original_video = entries[0]
-        self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
+        self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
     def test_youtube_toptracks(self):
         print('Skipping: The playlist page gives error 500')

From 040353942cc580337bbed09f536cbeb9c615a08d Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:10:35 +0100
Subject: [PATCH 033/394] Remove obsolete non-working test_youtube_toptracks

---
 test/test_youtube_lists.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index fae8a950a4b..69c5d52ebb5 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -43,15 +43,6 @@ def test_youtube_mix(self):
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
-    def test_youtube_toptracks(self):
-        print('Skipping: The playlist page gives error 500')
-        return
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
-        entries = result['entries']
-        self.assertEqual(len(entries), 100)
-
     def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()
         dl.params['extract_flat'] = True

From ad1356f29fbdb9add8b91d77ee1b8f9861dd56c4 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:44:57 +0000
Subject: [PATCH 034/394] Fix test_youtube_flat_playlist_extraction

---
 test/test_youtube_lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 69c5d52ebb5..07a6b6d06c5 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -52,7 +52,7 @@ def test_youtube_flat_playlist_extraction(self):
         entries = list(result['entries'])
         self.assertTrue(len(entries) == 1)
         video = entries[0]
-        self.assertEqual(video['_type'], 'url_transparent')
+        self.assertEqual(video['_type'], 'url')
         self.assertEqual(video['ie_key'], 'Youtube')
         self.assertEqual(video['id'], 'BaW_jenozKc')
         self.assertEqual(video['url'], 'BaW_jenozKc')

From 20fc43475289bd3335e5f96a8e0d8f5432f1d806 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:45:42 +0000
Subject: [PATCH 035/394] Unthrottle download speed by responding to "n"
 parameter challenge

---
 youtube_dl/extractor/youtube.py | 313 ++++++++++++++++++++++++++++++--
 1 file changed, 297 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc4bd4a77f9..b3085ccd4e7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,6 +2,12 @@
 
 from __future__ import unicode_literals
 
+# should probably have been in compat.py
+try:
+    from future_builtins import map
+except ImportError:
+    pass
+
 import itertools
 import json
 import os.path
@@ -1253,6 +1259,17 @@ def _extract_player_info(cls, player_url):
             raise ExtractorError('Cannot identify player %r' % player_url)
         return id_m.group('id')
 
+    def _get_player_code(self, video_id, player_url, player_id=None):
+        if not player_id:
+            player_id = self._extract_player_info(player_url)
+
+        if player_id not in self._code_cache:
+            self._code_cache[player_id] = self._download_webpage(
+                player_url, video_id,
+                note='Downloading player ' + player_id,
+                errnote='Download of %s failed' % player_url)
+        return self._code_cache[player_id]
+
     def _extract_signature_function(self, video_id, player_url, example_sig):
         player_id = self._extract_player_info(player_url)
 
@@ -1265,12 +1282,7 @@ def _extract_signature_function(self, video_id, player_url, example_sig):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
-        if player_id not in self._code_cache:
-            self._code_cache[player_id] = self._download_webpage(
-                player_url, video_id,
-                note='Downloading player ' + player_id,
-                errnote='Download of %s failed' % player_url)
-        code = self._code_cache[player_id]
+        code = self._get_player_code(video_id, player_url, player_id)
         res = self._parse_sig_js(code)
 
         test_string = ''.join(map(compat_chr, range(len(example_sig))))
@@ -1349,11 +1361,6 @@ def _decrypt_signature(self, s, video_id, player_url):
         if player_url is None:
             raise ExtractorError('Cannot decrypt signature without player_url')
 
-        if player_url.startswith('//'):
-            player_url = 'https:' + player_url
-        elif not re.match(r'https?://', player_url):
-            player_url = compat_urlparse.urljoin(
-                'https://www.youtube.com', player_url)
         try:
             player_id = (player_url, self._signature_cache_id(s))
             if player_id not in self._player_cache:
@@ -1370,6 +1377,280 @@ def _decrypt_signature(self, s, video_id, player_url):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
+    def _extract_player_url(self, webpage):
+        player_url = self._search_regex(
+            r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
+            webpage or '', 'player URL', fatal=False)
+        if not player_url:
+            return
+        if player_url.startswith('//'):
+            player_url = 'https:' + player_url
+        elif not re.match(r'https?://', player_url):
+            player_url = compat_urlparse.urljoin(
+                'https://www.youtube.com', player_url)
+        return player_url
+
+    # Based on an equivalent function [1] in the youtube.lua script from VLC
+    # Many thanks to @linkfanel [2]
+    # NB This code could fail if YT should revise the player code and would then have
+    # to be reworked (thankless task previously undertaken at [1])
+    # 1. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 2. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    def _n_descramble(self, n_param, js):
+        """Compute the response to YT's "n" parameter challenge
+
+        Args:
+        n_param -- challenge string that is the value of the
+                   URL's "n" query parameter
+        js      -- text of the JS player code that includes the
+                   challenge response algorithm
+        """
+        if not js:
+            return
+
+        # helper functions (part 1)
+        def isiterable(x):
+            try:
+                return x.__getitem__ and True
+            except AttributeError:
+                return False
+
+        def find_first(pattern, string, flags=0, groups=1):
+            pattern = re.compile(pattern, flags)
+            return next(map(lambda m: m.groups() if groups is True else m.group(groups),
+                            pattern.finditer(string)),
+                        (None, ) * pattern.groups if groups is True else None)
+
+        # Look for the descrambler function's name
+        # a.D&&(b=a.get("n"))&&(b=lha(b),a.set("n",b))}};
+        descrambler = find_first(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', js)
+        if not descrambler:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling function name")
+            return
+        # Fetch the code of the descrambler function
+        # lha=function(a){var b=a.split(""),c=[310282131,"KLf3",b,null,function(d,e){d.push(e)},-45817231, [data and transformations...] ,1248130556];c[3]=c;c[15]=c;c[18]=c;try{c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [scripted calls...] ,c[25](c[33],c[3])}catch(d){return"enhanced_except_4ZMBnuz-_w8_"+a}return b.join("")};
+        code = find_first(r'(?s)%s=function\([^)]+\)\{(.+?)\};' % (descrambler, ), js)
+        if not code:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling code")
+            return
+        # Split code into two main sections: 1/ data and transformations,
+        # and 2/ a script of calls
+        datac, script = find_first(r'(?s)c=\[(.+)\];.+?;try\{(.+)\}catch\(', code, groups=True)
+        if not datac or not script:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
+            return
+        # Split "n" parameter into a table as descrambling operates on it
+        # as one of several arrays - in Python just copy it as a list
+        n = list(n_param)
+        # Helper: table_len = function() ... end - in Python just use len
+
+        # Common routine shared by the compound transformations,
+        # compounding the "n" parameter with an input string,
+        # character by character using a Base64 alphabet.
+        # d.forEach(function(l,m,n){this.push(n[m]=h[(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length])},e.split(""))
+        def compound(ntab, strg, alphabet, charcode):
+            if ntab != n or type(strg) != compat_str:
+                return True
+            inp = list(strg)
+            llen = len(alphabet)
+            ntab_copy = ntab[:]
+            for i, c in enumerate(ntab_copy):
+                if type(c) != compat_str:
+                    return True
+                pos1 = alphabet.find(c)
+                pos2 = alphabet.find(inp[i])
+                if pos1 < 0 or pos2 < 0:
+                    return True
+                pos = (pos1 - pos2 + charcode - 32) % llen
+                newc = alphabet[pos]
+                ntab[i] = newc
+                inp.append(newc)
+
+        # The data section contains among others function code for a number
+        # of transformations, most of which are basic array operations.
+        # We can match these functions' code to identify them, and emulate
+        # the corresponding transformations.
+
+        # helper fns (in-place)
+        def swap(s, i, j):
+            x = s[i]
+            s[i] = s[j]
+            s[j] = x
+
+        def rotate(s, i):
+            tmp = s[:]
+            tmp[i:] = s
+            tmp[:i] = tmp[len(s):]
+            s[:] = tmp[:len(s)]
+
+        def remove(s, i):
+            del s[i]
+
+        # Compounding functions use a subfunction, so we need to be
+        # more specific in how much parsed data we consume.
+        cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
+        def_skip = r"(?s)^.*?\},\s*(.*)$"
+
+        trans = (
+            # fn_name, fn, fn_detect_pattern, skip_re
+            ('reverse', lambda tab, _: tab.reverse(),
+                        # noqa: E127
+                        # function(d){d.reverse()}
+                        # function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])}
+                        r"^function\(d\)",
+                        def_skip),
+            ('append', lambda tab, val: tab.append(val),
+                       # noqa: E127
+                       # function(d,e){d.push(e)}
+                       r"^function\(d,e\){d\.push\(e\)\},",
+                       def_skip),
+            ('remove', lambda tab, i: remove(tab, i % len(tab)) if type(i) == int else True,
+                       # noqa: E127
+                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)}
+                       r"^[^}]+?;d\.splice\(e,1\)\},",
+                       def_skip),
+            ('swap', lambda tab, i: swap(tab, 0, i % len(tab)) if type(i) == int else True,
+                     # noqa: E127
+                     # function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f}
+                     # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}
+                     r"^[^}]+?;(?:var\sf=d\[0\];d\[0\]=d\[e\];d\[e\]=f|d\.splice\(0,1,d\.splice\(e,1,d\[0\]\)\[0\]\))\},",
+                     def_skip),
+            ('rotate', lambda tab, shift: rotate(tab, shift % len(tab)) if type(shift) == int else True,
+                       # noqa: E127
+                       # function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())}
+                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
+                       r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
+                       def_skip),
+            # Compound transformations first build a variation of a
+            # Base64 alphabet, then in a common section, compound the
+            # "n" parameter with an input string, character by character.
+            ('compound1', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+                          # noqa: E127
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
+                          r"^[^}]+?case\s58:f=96;",
+                          cp_skip),
+            ('compound2', lambda tab, s: compound(tab, s, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", 96),
+                          # noqa: E127
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
+
+                          r"^[^}]+?case\s58:f(?:-=14|=44);",
+                          cp_skip),
+            # Fallback
+            ('unid', lambda _, __: self.report_warning("Couldn't apply unidentified YouTube video throttling parameter transformation, aborting descrambling") or True,
+                     # noqa: E127
+                     None,
+                     def_skip),
+        )
+        # The data section actually mixes input data, reference to the
+        # "n" parameter array, and self-reference to its own array, with
+        # transformation functions used to modify itself. We parse it
+        # as such into a table.
+        data = []
+        datac += ","
+        while datac:
+            # Transformation functions
+            if re.match(r"^function\(", datac):
+                name, el, _, skip = next(
+                    itertools.dropwhile(
+                        lambda x: x[2] is not None and not re.match(x[2], datac), trans))
+                datac = find_first(skip, datac)
+            # String input data
+            elif re.match(r'^"[^"]*",', datac):
+                el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
+            # Integer input data
+            elif re.match(r'^-?\d+,', datac):
+                el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
+                el = int(el)
+            # Reference to "n" parameter array
+            elif re.match('^b,', datac):
+                el = n
+                datac = find_first(r"(?s)^b,\s*(.*)$", datac)
+            # Replaced by self-reference to data array after its declaration
+            elif re.match('^null,', datac):
+                el = data
+                datac = find_first(r"(?s)^null,\s*(.*)$", datac)
+            else:
+                self.report_warning("Couldn't parse unidentified YouTube video throttling parameter descrambling data"
+                                    '\nNear: "%s"' % datac[:64])
+                el = False
+                # Lua tables can't contain nil values: Python can, but still use False
+                datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
+            data.append(el)
+
+        # Debugging helper to print data array elements
+        def prd(el, tab=None):
+            if not el:
+                return "???"
+            elif el == n:
+                return "n"
+            elif el == data:
+                return "data"
+            elif type(el) == compat_str:
+                return '"%s"' % (el, )
+            elif type(el) == int:
+                if isiterable(tab):
+                    return "%d -> %d" % (el, el % len(tab), )
+                return "%d" % (el, )
+            else:
+                for tr in trans:
+                    if el == tr[1]:
+                        return tr[0]
+            return repr(el)
+
+        # The script section contains a series of calls to elements of
+        # the data section array onto other elements of it: calls to
+        # transformations, with a reference to the data array itself or
+        # the "n" parameter array as first argument, and often input data
+        # as a second argument. We parse and emulate those calls to follow
+        # the descrambling script.
+        # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
+        for ifunc, itab, iarg in map(lambda m: m.groups(),
+                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\]|[^)]*?)\)", script)):
+            func = data[int(ifunc)]
+            tab = data[int(itab)]
+            arg = iarg and data[int(iarg)]
+            # Uncomment to debug transformation chain
+            # nprev = ''.join(n)
+            # dprev = ' '.join(map(prd, data))
+            # print(''.join(('"n" parameter transformation: ', prd(func), "(", prd(tab), (", " + prd(arg, tab)) if arg else '', ") ", ifunc, "(", itab, (", " + iarg) if iarg else "", ")")))
+            if not callable(func) or not isiterable(tab) or func(tab, arg):
+                self.report_warning("Invalid data type encountered during YouTube video throttling parameter descrambling transformation chain, aborting"
+                                    "\nCouldn't descramble YouTube throttling URL parameter: data transfer will be throttled")
+                self.report_warning("Couldn't process youtube video URL, please check for updates to this script")
+                break
+            # Uncomment to debug transformation chain
+            # nnew = ''.join(n)
+            # if nprev != nnew:
+            #     print('from: ' + nprev + "\nto:   " + nnew)
+            # dnew = ' '.join(map(prd, data))
+            # if dprev != dnew:
+            #     print('from: ' + dprev + "\nto:   " + dnew)
+        return ''.join(n)
+
+    def _unthrottle_format_urls(self, video_id, player_url, formats):
+        if not player_url:
+            return
+        player_id = self._extract_player_info(player_url)
+        code = self._get_player_code(video_id, player_url, player_id)
+        n_cache = {}
+        for fmt in formats:
+            parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
+            qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
+            n_param = qs.get('n')
+            if not n_param:
+                continue
+            n_param = n_param[-1]
+            n_response = n_cache.get(n_param)
+            if not n_response:
+                n_response = self._n_descramble(n_param, code)
+                if n_response:
+                    n_cache[n_param] = n_response
+            if n_response:
+                qs['n'] = [n_response]
+                fmt['url'] = compat_urlparse.urlunparse(
+                    parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
@@ -1631,11 +1912,7 @@ def feed_entry(name):
                 if not (sc and fmt_url and encrypted_sig):
                     continue
                 if not player_url:
-                    if not webpage:
-                        continue
-                    player_url = self._search_regex(
-                        r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
-                        webpage, 'player URL', fatal=False)
+                    player_url = self._extract_player_url(webpage)
                 if not player_url:
                     continue
                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
@@ -1781,6 +2058,10 @@ def feed_entry(name):
         is_live = video_details.get('isLive')
         owner_profile_url = microformat.get('ownerProfileUrl')
 
+        if not player_url:
+            player_url = self._extract_player_url(webpage)
+        self._unthrottle_format_urls(video_id, player_url, formats)
+
         info = {
             'id': video_id,
             'title': self._live_title(video_title) if is_live else video_title,

From 7109df0535757b38ccfcaf28d345f364673546fb Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Tue, 2 Nov 2021 11:18:39 +0000
Subject: [PATCH 036/394] Back-port JS interpreter upgrade from yt-dlp PR #1437

---
 test/test_jsinterp.py  |  51 +++++
 youtube_dl/compat.py   |   5 +
 youtube_dl/jsinterp.py | 504 ++++++++++++++++++++++++++++++++---------
 3 files changed, 453 insertions(+), 107 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c24b8ca742a..4d05ea61013 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -112,6 +112,57 @@ def test_call(self):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        # function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..24dd7d78a62 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,10 @@
 import sys
 import xml.etree.ElementTree
 
+try:
+    import collections.abc as compat_collections_abc
+except ImportError:
+    import collections as compat_collections_abc
 
 try:
     import urllib.request as compat_urllib_request
@@ -3006,6 +3010,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_b64decode',
     'compat_basestring',
     'compat_chr',
+    'compat_collections_abc',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7bda596102a..061e92c2a4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -8,6 +8,15 @@
     ExtractorError,
     remove_quotes,
 )
+from .compat import (
+    compat_collections_abc
+)
+MutableMapping = compat_collections_abc.MutableMapping
+
+
+class Nonlocal:
+    pass
+
 
 _OPERATORS = [
     ('|', operator.or_),
@@ -22,11 +31,55 @@
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
 
+class JS_Break(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid break')
+
+
+class JS_Continue(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid continue')
+
+
+class LocalNameSpace(MutableMapping):
+    def __init__(self, *stack):
+        self.stack = tuple(stack)
+
+    def __getitem__(self, key):
+        for scope in self.stack:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        for scope in self.stack:
+            if key in scope:
+                scope[key] = value
+                break
+        else:
+            self.stack[0][key] = value
+        return value
+
+    def __delitem__(self, key):
+        raise NotImplementedError('Deleting is not supported')
+
+    def __iter__(self):
+        for scope in self.stack:
+            for scope_item in iter(scope):
+                yield scope_item
+
+    def __len__(self, key):
+        return len(iter(self))
+
+    def __repr__(self):
+        return 'LocalNameSpace%s' % (self.stack, )
+
+
 class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
@@ -34,11 +87,58 @@ def __init__(self, code, objects=None):
         self.code = code
         self._functions = {}
         self._objects = objects
+        self.__named_object_counter = 0
+
+    def _named_object(self, namespace, obj):
+        self.__named_object_counter += 1
+        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        namespace[name] = obj
+        return name
+
+    @staticmethod
+    def _separate(expr, delim=',', max_split=None):
+        if not expr:
+            return
+        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
+        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        for idx, char in enumerate(expr):
+            if char in parens:
+                parens[char] += 1
+            is_in_parens = (parens['['] - parens[']']
+                            or parens['('] - parens[')']
+                            or parens['{'] - parens['}'])
+            if char == delim[pos] and not is_in_parens:
+                if pos == max_pos:
+                    pos = 0
+                    yield expr[start: idx - max_pos]
+                    start = idx + 1
+                    splits += 1
+                    if max_split and splits >= max_split:
+                        break
+                else:
+                    pos += 1
+            else:
+                pos = 0
+        yield expr[start:]
+
+    @staticmethod
+    def _separate_at_paren(expr, delim):
+        separated = list(JSInterpreter._separate(expr, delim, 1))
+        if len(separated) < 2:
+            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+        return separated[0][1:].strip(), separated[1].strip()
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
+        sub_statements = list(self._separate(stmt, ';'))
+        stmt = (sub_statements or ['']).pop()
+        for sub_stmt in sub_statements:
+            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+
         should_abort = False
         stmt = stmt.lstrip()
         stmt_m = re.match(r'var\s', stmt)
@@ -61,25 +161,119 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr == '':  # Empty expression
             return None
 
+        if expr.startswith('{'):
+            inner, outer = self._separate_at_paren(expr, '}')
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            if not outer or should_abort:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
         if expr.startswith('('):
-            parens_count = 0
-            for m in re.finditer(r'[()]', expr):
-                if m.group(0) == '(':
-                    parens_count += 1
+            inner, outer = self._separate_at_paren(expr, ')')
+            inner = self.interpret_expression(inner, local_vars, allow_recursion)
+            if not outer:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('['):
+            inner, outer = self._separate_at_paren(expr, ']')
+            name = self._named_object(local_vars, [
+                self.interpret_expression(item, local_vars, allow_recursion)
+                for item in self._separate(inner)])
+            expr = name + outer
+
+        m = re.match(r'try\s*', expr)
+        if m:
+            if expr[m.end()] == '{':
+                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            else:
+                try_expr, expr = expr[m.end() - 1:], ''
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('catch'):
+            # We ignore the catch block
+            _, expr = self._separate_at_paren(expr, '}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('for'):
+            def raise_constructor_error(c):
+                raise ExtractorError(
+                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
+
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            if remaining.startswith('{'):
+                body, expr = self._separate_at_paren(remaining, '}')
+            else:
+                m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if m:
+                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                    body, expr = self._separate_at_paren(remaining, '}')
+                    body = 'switch(%s){%s}' % (switch_val, body)
                 else:
-                    parens_count -= 1
-                    if parens_count == 0:
-                        sub_expr = expr[1:m.start()]
-                        sub_result = self.interpret_expression(
-                            sub_expr, local_vars, allow_recursion)
-                        remaining_expr = expr[m.end():].strip()
-                        if not remaining_expr:
-                            return sub_result
-                        else:
-                            expr = json.dumps(sub_result) + remaining_expr
+                    body, expr = remaining, ''
+            start, cndn, increment = self._separate(constructor, ';')
+            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
+                raise_constructor_error(constructor)
+            while True:
+                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                    break
+                try:
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    if should_abort:
+                        return ret
+                except JS_Break:
+                    break
+                except JS_Continue:
+                    pass
+                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
+                    raise_constructor_error(constructor)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('switch'):
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
+            body, expr = self._separate_at_paren(remaining, '}')
+            body, default = body.split('default:') if 'default:' in body else (body, None)
+            items = body.split('case ')[1:]
+            if default:
+                items.append('default:%s' % (default, ))
+            matched = False
+            for item in items:
+                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
+                if matched:
+                    try:
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        if should_abort:
+                            return ret
+                    except JS_Break:
                         break
-            else:
-                raise ExtractorError('Premature end of parens in %r' % expr)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        # Comma separated statements
+        sub_expressions = list(self._separate(expr))
+        expr = sub_expressions.pop().strip() if sub_expressions else ''
+        for sub_expr in sub_expressions:
+            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+
+        for m in re.finditer(r'''(?x)
+                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
+                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+            var = m.group('var1') or m.group('var2')
+            start, end = m.span()
+            sign = m.group('pre_sign') or m.group('post_sign')
+            ret = local_vars[var]
+            local_vars[var] += 1 if sign[0] == '+' else -1
+            if m.group('pre_sign'):
+                ret = local_vars[var]
+            expr = expr[:start] + json.dumps(ret) + expr[end:]
 
         for op, opfunc in _ASSIGN_OPERATORS:
             m = re.match(r'''(?x)
@@ -88,14 +282,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
                 (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
             if not m:
                 continue
-            right_val = self.interpret_expression(
-                m.group('expr'), local_vars, allow_recursion - 1)
+            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
 
             if m.groupdict().get('index'):
                 lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(
-                    m.group('index'), local_vars, allow_recursion)
-                assert isinstance(idx, int)
+                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+                if not isinstance(idx, int):
+                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -109,8 +302,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr.isdigit():
             return int(expr)
 
+        if expr == 'break':
+            raise JS_Break()
+        elif expr == 'continue':
+            raise JS_Continue()
+
         var_m = re.match(
-            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
             expr)
         if var_m:
             return local_vars[var_m.group('name')]
@@ -124,91 +322,161 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
             r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
+            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
             return val[idx]
 
+        def raise_expr_error(where, op, exp):
+            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+
+        for op, opfunc in _OPERATORS:
+            separated = list(self._separate(expr, op))
+            if len(separated) < 2:
+                continue
+            right_val = separated.pop()
+            left_val = op.join(separated)
+            left_val, should_abort = self.interpret_statement(
+                left_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('left-side', op, expr)
+            right_val, should_abort = self.interpret_statement(
+                right_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('right-side', op, expr)
+            return opfunc(left_val or 0, right_val)
+
         m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = remove_quotes(m.group('member') or m.group('member2'))
-            arg_str = m.group('args')
+            nl = Nonlocal()
 
-            if variable in local_vars:
-                obj = local_vars[variable]
-            else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
-
-            if arg_str is None:
-                # Member access
-                if member == 'length':
-                    return len(obj)
-                return obj[member]
-
-            assert expr.endswith(')')
-            # Function call
-            if arg_str == '':
-                argvals = tuple()
+            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+            arg_str = expr[m.end():]
+            if arg_str.startswith('('):
+                arg_str, remaining = self._separate_at_paren(arg_str, ')')
             else:
-                argvals = tuple([
+                arg_str, remaining = None, arg_str
+
+            def assertion(cndn, msg):
+                """ assert, but without risk of getting optimized out """
+                if not cndn:
+                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+
+            def eval_method():
+                # nonlocal member
+                member = nl.member
+                if variable == 'String':
+                    obj = str
+                elif variable in local_vars:
+                    obj = local_vars[variable]
+                else:
+                    if variable not in self._objects:
+                        self._objects[variable] = self.extract_object(variable)
+                    obj = self._objects[variable]
+
+                if arg_str is None:
+                    # Member access
+                    if member == 'length':
+                        return len(obj)
+                    return obj[member]
+
+                # Function call
+                argvals = [
                     self.interpret_expression(v, local_vars, allow_recursion)
-                    for v in arg_str.split(',')])
-
-            if member == 'split':
-                assert argvals == ('',)
-                return list(obj)
-            if member == 'join':
-                assert len(argvals) == 1
-                return argvals[0].join(obj)
-            if member == 'reverse':
-                assert len(argvals) == 0
-                obj.reverse()
-                return obj
-            if member == 'slice':
-                assert len(argvals) == 1
-                return obj[argvals[0]:]
-            if member == 'splice':
-                assert isinstance(obj, list)
-                index, howMany = argvals
-                res = []
-                for i in range(index, min(index + howMany, len(obj))):
-                    res.append(obj.pop(index))
-                return res
-
-            return obj[member](argvals)
-
-        for op, opfunc in _OPERATORS:
-            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
-            if not m:
-                continue
-            x, abort = self.interpret_statement(
-                m.group('x'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature left-side return of %s in %r' % (op, expr))
-            y, abort = self.interpret_statement(
-                m.group('y'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature right-side return of %s in %r' % (op, expr))
-            return opfunc(x, y)
+                    for v in self._separate(arg_str)]
+
+                if obj == str:
+                    if member == 'fromCharCode':
+                        assertion(argvals, 'takes one or more arguments')
+                        return ''.join(map(chr, argvals))
+                    raise ExtractorError('Unsupported string method %s' % (member, ))
+
+                if member == 'split':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(argvals == [''], 'with arguments is not implemented')
+                    return list(obj)
+                elif member == 'join':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return argvals[0].join(obj)
+                elif member == 'reverse':
+                    assertion(not argvals, 'does not take any arguments')
+                    obj.reverse()
+                    return obj
+                elif member == 'slice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return obj[argvals[0]:]
+                elif member == 'splice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    index, howMany = (argvals + [len(obj)])[:2]
+                    if index < 0:
+                        index += len(obj)
+                    add_items = argvals[2:]
+                    res = []
+                    for i in range(index, min(index + howMany, len(obj))):
+                        res.append(obj.pop(index))
+                    for i, item in enumerate(add_items):
+                        obj.insert(index + i, item)
+                    return res
+                elif member == 'unshift':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    for item in reversed(argvals):
+                        obj.insert(0, item)
+                    return obj
+                elif member == 'pop':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(not argvals, 'does not take any arguments')
+                    if not obj:
+                        return
+                    return obj.pop()
+                elif member == 'push':
+                    assertion(argvals, 'takes one or more arguments')
+                    obj.extend(argvals)
+                    return obj
+                elif member == 'forEach':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    f, this = (argvals + [''])[:2]
+                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                elif member == 'indexOf':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    idx, start = (argvals + [0])[:2]
+                    try:
+                        return obj.index(idx, start)
+                    except ValueError:
+                        return -1
+
+                if isinstance(obj, list):
+                    member = int(member)
+                    nl.member = member
+                return obj[member](argvals)
+
+            if remaining:
+                return self.interpret_expression(
+                    self._named_object(local_vars, eval_method()) + remaining,
+                    local_vars, allow_recursion)
+            else:
+                return eval_method()
 
-        m = re.match(
-            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
+        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
-                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
-            if fname not in self._functions:
+                for v in self._separate(m.group('args'))])
+            if fname in local_vars:
+                return local_vars[fname](argvals)
+            elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
-        raise ExtractorError('Unsupported JS expression %r' % expr)
+        if expr:
+            raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -233,30 +501,52 @@ def extract_object(self, objname):
 
         return obj
 
-    def extract_function(self, funcname):
+    def extract_function_code(self, funcname):
+        """ @returns argnames, code """
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
-                \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
+                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
             self.code)
+        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
-        argnames = func_m.group('args').split(',')
+        return func_m.group('args').split(','), code
 
-        return self.build_function(argnames, func_m.group('code'))
+    def extract_function(self, funcname):
+        return self.extract_function_from_code(*self.extract_function_code(funcname))
+
+    def extract_function_from_code(self, argnames, code, *global_stack):
+        local_vars = {}
+        while True:
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            if mobj is None:
+                break
+            start, body_start = mobj.span()
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(
+                local_vars,
+                self.extract_function_from_code(
+                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    body, local_vars, *global_stack))
+            code = code[:start] + name + remaining
+        return self.build_function(argnames, code, local_vars, *global_stack)
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
-        return f(args)
-
-    def build_function(self, argnames, code):
-        def resf(args):
-            local_vars = dict(zip(argnames, args))
-            for stmt in code.split(';'):
-                res, abort = self.interpret_statement(stmt, local_vars)
-                if abort:
+        return self.extract_function(funcname)(args)
+
+    def build_function(self, argnames, code, *global_stack):
+        global_stack = list(global_stack) or [{}]
+        local_vars = global_stack.pop(0)
+
+        def resf(args, **kwargs):
+            local_vars.update(dict(zip(argnames, args)))
+            local_vars.update(kwargs)
+            var_stack = LocalNameSpace(local_vars, *global_stack)
+            for stmt in self._separate(code.replace('\n', ''), ';'):
+                ret, should_abort = self.interpret_statement(stmt, var_stack)
+                if should_abort:
                     break
-            return res
+            return ret
         return resf

From a3aaca9c32fbb50d84a987a836bcd7fa8f9246ba Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 4 Nov 2021 12:48:06 +0000
Subject: [PATCH 037/394] Handle default in switch better

Add https://github.com/yt-dlp/yt-dlp/commit/a1fc7ca0743c8df06416e68ee74b64e07dfe7135
Thanks @coletdjnz
---
 test/test_jsinterp.py  | 15 +++++++++++++++
 youtube_dl/jsinterp.py | 23 ++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d05ea61013..acdabffb1f8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -133,6 +133,21 @@ def test_switch(self):
         self.assertEqual(jsi.call_function('x', 3), 6)
         self.assertEqual(jsi.call_function('x', 5), 0)
 
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
     def test_try(self):
         jsi = JSInterpreter('''
         function x() { try{return 10} catch(e){return 5} }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 061e92c2a4a..c35765702a6 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,21 +240,26 @@ def raise_constructor_error(c):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
-            body, default = body.split('default:') if 'default:' in body else (body, None)
-            items = body.split('case ')[1:]
-            if default:
-                items.append('default:%s' % (default, ))
-            matched = False
-            for item in items:
-                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
-                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
-                if matched:
+            items = body.replace('default:', 'case default:').split('case ')[1:]
+            for default in (False, True):
+                matched = False
+                for item in items:
+                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    if default:
+                        matched = matched or case == 'default'
+                    elif not matched:
+                        matched = (case != 'default'
+                                   and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
+                    if not matched:
+                        continue
                     try:
                         ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
+                if matched:
+                    break
             return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
 
         # Comma separated statements

From 8e069597c658810567ced5f8046dc5d14ab93037 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sat, 13 Nov 2021 03:05:38 +0000
Subject: [PATCH 038/394] Update for player 8d287e4di

3-arg function, compound as a transformation
---
 youtube_dl/extractor/youtube.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b3085ccd4e7..11845114ac0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1390,12 +1390,13 @@ def _extract_player_url(self, webpage):
                 'https://www.youtube.com', player_url)
         return player_url
 
-    # Based on an equivalent function [1] in the youtube.lua script from VLC
-    # Many thanks to @linkfanel [2]
+    # Based on an equivalent function [2] in the youtube.lua script from VLC
+    # Many thanks to @linkfanel [3]
     # NB This code could fail if YT should revise the player code and would then have
-    # to be reworked (thankless task previously undertaken at [1])
-    # 1. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
-    # 2. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    # to be reworked (thankless task previously undertaken at [1] and [2])
+    # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
+    # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
     def _n_descramble(self, n_param, js):
         """Compute the response to YT's "n" parameter challenge
 
@@ -1521,6 +1522,11 @@ def remove(s, i):
                        # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
                        r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
                        def_skip),
+            ('compound', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+                         # noqa: E127
+                         # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
+                         r"^function\(\w,\w,\w\)",
+                         cp_skip),
             # Compound transformations first build a variation of a
             # Base64 alphabet, then in a common section, compound the
             # "n" parameter with an input string, character by character.
@@ -1533,7 +1539,6 @@ def remove(s, i):
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
-
                           r"^[^}]+?case\s58:f(?:-=14|=44);",
                           cp_skip),
             # Fallback
@@ -1606,7 +1611,8 @@ def prd(el, tab=None):
         # the descrambling script.
         # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
         for ifunc, itab, iarg in map(lambda m: m.groups(),
-                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\]|[^)]*?)\)", script)):
+                                     # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
+                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[\d+\]\(\))?[^)]*?\)", script)):
             func = data[int(ifunc)]
             tab = data[int(itab)]
             arg = iarg and data[int(iarg)]

From 383217bcfffaf7474afcc0b996db17571aa5552e Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 039/394] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 11845114ac0..6d565719ae9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From 0bb31529920240940c05be941df0978bca14b21e Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 13 Nov 2021 19:52:59 +0100
Subject: [PATCH 040/394] cleaning

---
 Makefile               | 14 +++++---------
 docs/supportedsites.md |  2 --
 youtube_dl/version.py  |  2 +-
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 3e17365b83d..eaa1f910969 100644
--- a/Makefile
+++ b/Makefile
@@ -5,11 +5,11 @@ clean:
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 
-PREFIX ?= /usr/local
-BINDIR ?= $(PREFIX)/bin
-MANDIR ?= $(PREFIX)/man
-SHAREDIR ?= $(PREFIX)/share
-PYTHON ?= /usr/bin/env python
+PREFIX = /Users/glen/local/
+BINDIR = $(PREFIX)/bin
+MANDIR = $(PREFIX)/share/man
+SHAREDIR = $(PREFIX)/share
+PYTHON = /usr/bin/env python3
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
 SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi)
@@ -24,10 +24,6 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
 	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
-	install -d $(DESTDIR)$(SHAREDIR)/zsh/site-functions
-	install -m 644 youtube-dl.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dl
-	install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
-	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 codetest:
 	flake8 .
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ed0d5e9d990..ae2a6b8b02b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -472,8 +472,6 @@
  - **LinuxAcademy**
  - **LiTV**
  - **LiveJournal**
- - **LiveLeak**
- - **LiveLeakEmbed**
  - **livestream**
  - **livestream:original**
  - **LnkGo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 461dd87cafe..509aeb19ef1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.06.06'
+__version__ = '2021.11.13'

From d9a4cdf102bb21daf384b5664b89534814da59bb Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 13 Nov 2021 20:16:40 +0100
Subject: [PATCH 041/394] cleaning

---
 Makefile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index eaa1f910969..3a7bb71c14f 100644
--- a/Makefile
+++ b/Makefile
@@ -18,12 +18,10 @@ SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then ech
 MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi)
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
-	install -d $(DESTDIR)$(BINDIR)
-	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
+	install -d $(BINDIR)
+	install -m 755 youtube-dl $(BINDIR)
 	install -d $(DESTDIR)$(MANDIR)/man1
 	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
-	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
-	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
 
 codetest:
 	flake8 .

From df2e302388bbb3a9788bbd8c55d1f0d69c8db1d2 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 13 Nov 2021 20:24:36 +0100
Subject: [PATCH 042/394] cleaning

---
 Makefile | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 3a7bb71c14f..3d2e2ca72c1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
@@ -17,11 +17,11 @@ SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then ech
 # set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2
 MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi)
 
-install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(BINDIR)
 	install -m 755 youtube-dl $(BINDIR)
 	install -d $(DESTDIR)$(MANDIR)/man1
-	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
+	install -m 644 youtube-dl.1 $(MANDIR)/man1
 
 codetest:
 	flake8 .
@@ -47,7 +47,7 @@ offlinetest: codetest
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites
+.PHONY: all clean install test tar bash-completion pypi-files ot offlinetest codetest supportedsites
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
@@ -95,23 +95,13 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co
 
 bash-completion: youtube-dl.bash-completion
 
-youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
-	$(PYTHON) devscripts/zsh-completion.py
-
-zsh-completion: youtube-dl.zsh
-
-youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
-	$(PYTHON) devscripts/fish-completion.py
-
-fish-completion: youtube-dl.fish
-
 lazy-extractors: youtube_dl/extractor/lazy_extractors.py
 
 _EXTRACTOR_FILES = $(shell find youtube_dl/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py')
 youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion  ChangeLog AUTHORS
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -125,5 +115,5 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		bin devscripts test youtube_dl docs \
 		ChangeLog AUTHORS LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
-		youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \
+		setup.py setup.cfg \
 		youtube-dl

From 7d1059d6b8c817cc37d7598c7372ba832e6b55e2 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sat, 13 Nov 2021 22:03:19 +0000
Subject: [PATCH 043/394] Don't find classname as part of class="...
 x-classname ...", etc

Eg, in [1], the class with name 'plist-info' was found when searching for 'info'.

1. https://github.com/ytdl-org/youtube-dl/issues/30230
---
 test/test_utils.py  |  5 ++++-
 youtube_dl/utils.py | 12 ++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 50fb5f10147..7c6a382fd9a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1463,7 +1463,10 @@ def test_get_element_by_attribute(self):
 
     def test_get_elements_by_class(self):
         html = '''
-            <span class="foo bar">nice</span><span class="foo bar">also nice</span>
+            <span class="not-foo bar">nasty</span>
+            <span class="foo bar">nice</span>
+            <span class="bar foo">"also nice"</span>
+            <span class="bar foo-impostor">also nasty</span>
         '''
 
         self.assertEqual(get_elements_by_class('foo', html), ['nice', 'also nice'])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fcb78757032..0bc6be5094b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1962,7 +1962,9 @@ def get_element_by_attribute(attribute, value, html, escape_value=True):
 def get_elements_by_class(class_name, html):
     """Return the content of all tags with the specified class in the passed HTML document as a list"""
     return get_elements_by_attribute(
-        'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
+        # class names can contain alphanumeric, -, _ and \ for escapes
+        # don't allow a word break at -
+        'class', r'(?:[\w\s\\-]*?[\w\s])?\b%s\b(?:[\w\s\\][\w\s\\-]*?)?' % re.escape(class_name),
         html, escape_value=False)
 
 
@@ -1973,11 +1975,13 @@ def get_elements_by_attribute(attribute, value, html, escape_value=True):
 
     retlist = []
     for m in re.finditer(r'''(?xs)
-        <([a-zA-Z0-9:._-]+)
+        <([a-zA-Z0-9:._-]+)	# conservative pattern: HTML tags don't have :._-
+         # (?:\s[^>]+)          # this seems to be simpler than the below and work the same?
          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
-         \s+%s=['"]?%s['"]?
+         \s*\b%s\s*=\s*(?P<__q>'|"|\b)%s(?P=__q)
+         # (?:\s[^>]+)?         # as above
          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
-        \s*>
+         \s*>
         (?P<content>.*?)
         </\1>
     ''' % (re.escape(attribute), value), html):

From 7ce02b5bf95cb4b1bac9b23344dbcabace5f9511 Mon Sep 17 00:00:00 2001
From: Conner <65339198+connercsbn@users.noreply.github.com>
Date: Sun, 14 Nov 2021 16:00:20 -0800
Subject: [PATCH 044/394] [Podchaser] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/podchaser.py  | 98 ++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/podchaser.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..eaa78b16dff 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -929,6 +929,7 @@
     PluralsightIE,
     PluralsightCourseIE,
 )
+from .podchaser import PodchaserIE
 from .podomatic import PodomaticIE
 from .pokemon import PokemonIE
 from .polskieradio import (
diff --git a/youtube_dl/extractor/podchaser.py b/youtube_dl/extractor/podchaser.py
new file mode 100644
index 00000000000..80f970c1b2b
--- /dev/null
+++ b/youtube_dl/extractor/podchaser.py
@@ -0,0 +1,98 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+from datetime import datetime
+from ..utils import float_or_none, try_get, str_to_int
+from ..compat import compat_str
+from .common import InfoExtractor
+
+
+class PodchaserIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?podchaser\.com/
+        (?:(?:podcasts)|(?:creators))
+        /[\w\-]+-
+        (?:
+            (?P<creator_id>[\d]+[\w]+)|(?P<podcast_id>[\d]+))
+        (?:/episodes/[\w\-]+-
+            (?P<id>[\d]+))?'''
+
+    _TESTS = [{
+        'url': 'https://www.podchaser.com/podcasts/cum-town-36924/episodes/ep-285-freeze-me-off-104365585',
+        'info_dict': {
+            'id': '104365585',
+            'title': "Ep. 285 – freeze me off",
+            'description': 'cam ahn',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'ext': 'mp3',
+            'categories': ['Comedy'],
+            'series': 'Cum Town',
+            'duration': 3708,
+            'timestamp': 1636560059,
+            'upload_date': '20211110'
+        }
+    }, {
+        'url': 'https://www.podchaser.com/podcasts/the-bone-zone-28853',
+        'info_dict': {
+            'id': '28853',
+            'title': 'The Bone Zone',
+            'description': 'md5:c39acd897170a8bf3ad94fc45dc25060',
+        },
+        'playlist_count': 6
+    }, {
+        'url': 'https://www.podchaser.com/creators/todd-glass-107ZzkFiEQ',
+        'info_dict': {
+            'id': '107ZzkFiEQ',
+            'title': 'Todd Glass',
+            'description': 'md5:f0302588c98d474d4e8551f4914042c9',
+        },
+        'playlist_mincount': 48
+    }, {
+        'url': 'https://www.podchaser.com/podcasts/sean-carrolls-mindscape-scienc-699349/episodes',
+        'info_dict': {
+            'id': '699349',
+            'title': "Sean Carroll's Mindscape: Science, Society, Philosophy, Culture, Arts, and Ideas",
+            'description': 'md5:a1ed5db77378a736b3a85a2331cbc05a'
+        },
+        'playlist_count': 25
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        audio_id, podcast_id, creator_id = compat_str(mobj.group('id')), \
+            compat_str(mobj.group('podcast_id')), compat_str(mobj.group('creator_id'))
+        webpage = self._download_webpage(url, audio_id)
+        page_title = self._html_search_meta(['title', 'og:title', 'twitter:title'], webpage, default=None) \
+            or self._search_regex(
+                r'<h1[^>]*>(.+?)</h1>', webpage, 'title', fatal=False, default="Podchaser Podcast")
+        page_description = self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage, default=None)
+
+        data = self._search_regex(
+            r'window.__APP_STATE__\s*=\s*(["\']?{.+?}["\']?);</script>', webpage, 'app state')
+
+        while isinstance(data, compat_str):
+            data = self._parse_json(data, audio_id)
+
+        episodes = try_get(data, lambda x: x['podcast']['episodes']['entities'], dict) or {}
+        episode_list = [(episodes.get(episode_id), episode_id) for episode_id in episodes]
+
+        entries = [{
+            'id': episode_id,
+            'title': episode.get('title'),
+            'description': episode.get('description'),
+            'url': episode.get('audio_url'),
+            'thumbnail': try_get(episode, lambda x: x['podcast']['image_url']),
+            'duration': str_to_int(episode.get('length')),
+            'timestamp': datetime.strptime(episode.get('air_date'), '%Y-%m-%d %H:%M:%S').timestamp(),
+            'rating': float_or_none(episode.get('rating')),
+            'categories': [x['text'] for x in try_get(episode, lambda x: x['podcast']['categories'], list) or []],
+            'tags': [tag['text'] for tag in episode.get('tags') or []],
+            'series': try_get(episode, lambda x: x['podcast']['title'], compat_str),
+        } for episode, episode_id in episode_list]
+
+        if len(entries) > 1:
+            return self.playlist_result(
+                entries, playlist_id=(creator_id or podcast_id), playlist_title=page_title,
+                playlist_description=page_description)
+
+        return entries[0]

From b54c6a6b0f9a8f1b2b30bb459d00b80dc62ca0d2 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 15 Nov 2021 06:58:20 +0100
Subject: [PATCH 045/394] cleaning

---
 youtube_dl/extractor/motherless.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index ef1e081f20e..3e244caf1cc 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -127,7 +127,7 @@ def _real_extract(self, url):
 
         comment_count = webpage.count('class="media-comment-contents"')
         uploader_id = self._html_search_regex(
-            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
+            r'<span class=\"username\">([^"]+)<\/span>',
             webpage, 'uploader_id')
 
         categories = self._html_search_meta('keywords', webpage, default=None)

From c084110eed143f6d6b59356f1da40dbbfac6b21a Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 15 Nov 2021 07:14:48 +0100
Subject: [PATCH 046/394] cleaning

---
 test/test_jsinterp.py  | 58 ++++++++++++++++++++++++++++++++++++++++++
 youtube_dl/jsinterp.py |  3 +--
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c24b8ca742a..bbbf330f319 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -112,6 +112,64 @@ def test_call(self):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7bda596102a..8ac2b7b7a39 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -248,8 +248,7 @@ def extract_function(self, funcname):
         return self.build_function(argnames, func_m.group('code'))
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
-        return f(args)
+        return self.extract_function(funcname)(args)
 
     def build_function(self, argnames, code):
         def resf(args):

From 9db547ddd0dd8d9afbc3df1c0644acd0eecf2310 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sat, 13 Nov 2021 03:05:38 +0000
Subject: [PATCH 047/394] Update for player 8d287e4d

3-arg function, compound as a transformation
---
 youtube_dl/extractor/youtube.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b3085ccd4e7..11845114ac0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1390,12 +1390,13 @@ def _extract_player_url(self, webpage):
                 'https://www.youtube.com', player_url)
         return player_url
 
-    # Based on an equivalent function [1] in the youtube.lua script from VLC
-    # Many thanks to @linkfanel [2]
+    # Based on an equivalent function [2] in the youtube.lua script from VLC
+    # Many thanks to @linkfanel [3]
     # NB This code could fail if YT should revise the player code and would then have
-    # to be reworked (thankless task previously undertaken at [1])
-    # 1. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
-    # 2. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    # to be reworked (thankless task previously undertaken at [1] and [2])
+    # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
+    # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
     def _n_descramble(self, n_param, js):
         """Compute the response to YT's "n" parameter challenge
 
@@ -1521,6 +1522,11 @@ def remove(s, i):
                        # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
                        r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
                        def_skip),
+            ('compound', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+                         # noqa: E127
+                         # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
+                         r"^function\(\w,\w,\w\)",
+                         cp_skip),
             # Compound transformations first build a variation of a
             # Base64 alphabet, then in a common section, compound the
             # "n" parameter with an input string, character by character.
@@ -1533,7 +1539,6 @@ def remove(s, i):
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
-
                           r"^[^}]+?case\s58:f(?:-=14|=44);",
                           cp_skip),
             # Fallback
@@ -1606,7 +1611,8 @@ def prd(el, tab=None):
         # the descrambling script.
         # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
         for ifunc, itab, iarg in map(lambda m: m.groups(),
-                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\]|[^)]*?)\)", script)):
+                                     # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
+                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[\d+\]\(\))?[^)]*?\)", script)):
             func = data[int(ifunc)]
             tab = data[int(itab)]
             arg = iarg and data[int(iarg)]

From 9ba0be37ecea1601410022ce0811d33097998eaf Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 048/394] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c5e3a300fe9..c3380716ffc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From 58bfb65a62593ac7b3fae38ffd2040d5c5e2da04 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 16 Nov 2021 05:49:44 +0000
Subject: [PATCH 049/394] Update for player 2dfe380c

Ad hoc update of c[] array after parsing core descrambler
Use specified b64 alphabet with compound(,,alphabet,), not guess
---
 youtube_dl/extractor/youtube.py | 51 +++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 11845114ac0..c5e3a300fe9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1436,10 +1436,18 @@ def find_first(pattern, string, flags=0, groups=1):
             return
         # Split code into two main sections: 1/ data and transformations,
         # and 2/ a script of calls
-        datac, script = find_first(r'(?s)c=\[(.+)\];.+?;try\{(.+)\}catch\(', code, groups=True)
+        datac, xc, script = find_first(r'(?s)c=\[(.+)\];(.+?);try\{(.+)\}catch\(', code, groups=True)
         if not datac or not script:
             self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
             return
+        if xc:
+            # ad hoc update of c[] array after declaration (player 2dfe380c, ...)
+            xc = [int(x) for x in re.findall(r'c\[(\d+)\]=c\b', xc)]
+            if not xc:
+                self.report_warning("Couldn't extract YouTube video throttling parameter additional rules")
+        else:
+            xc = []
+
         # Split "n" parameter into a table as descrambling operates on it
         # as one of several arrays - in Python just copy it as a list
         n = list(n_param)
@@ -1487,6 +1495,11 @@ def rotate(s, i):
         def remove(s, i):
             del s[i]
 
+        b64_alphabets = [
+            "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
+        ]
+
         # Compounding functions use a subfunction, so we need to be
         # more specific in how much parsed data we consume.
         cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
@@ -1522,20 +1535,31 @@ def remove(s, i):
                        # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
                        r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
                        def_skip),
-            ('compound', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+            ('alphabet1', lambda: b64_alphabets[0],
+                          # noqa: E127
+                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
+                          r"^[^}]+?case\s58:d=96;.+?\}return",
+                          def_skip),
+            ('alphabet2', lambda: b64_alphabets[1],
+                          # noqa: E127
+                          # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
+                          r"^[^}]+?case\s58:d=44;[^}]+\}return",
+                          def_skip),
+            ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
                          # noqa: E127
                          # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
                          r"^function\(\w,\w,\w\)",
                          cp_skip),
+
             # Compound transformations first build a variation of a
             # Base64 alphabet, then in a common section, compound the
             # "n" parameter with an input string, character by character.
-            ('compound1', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+            ('compound1', lambda tab, s: compound(tab, s, b64_alphabets[0], 96),
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
                           r"^[^}]+?case\s58:f=96;",
                           cp_skip),
-            ('compound2', lambda tab, s: compound(tab, s, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", 96),
+            ('compound2', lambda tab, s: compound(tab, s, b64_alphabets[1], 96),
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
@@ -1583,6 +1607,10 @@ def remove(s, i):
                 datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
             data.append(el)
 
+        # Additional rules
+        for idx in xc:
+            data[idx] = data
+
         # Debugging helper to print data array elements
         def prd(el, tab=None):
             if not el:
@@ -1610,12 +1638,19 @@ def prd(el, tab=None):
         # as a second argument. We parse and emulate those calls to follow
         # the descrambling script.
         # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
-        for ifunc, itab, iarg in map(lambda m: m.groups(),
-                                     # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
-                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[\d+\]\(\))?[^)]*?\)", script)):
-            func = data[int(ifunc)]
+        for ifunc, itab, iarg, iextra in map(lambda m: m.groups(),
+                                             # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
+                                             re.finditer(
+                                                 r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[(\d+)\]\(\))?[^)]*?\)",
+                                                 script)):
             tab = data[int(itab)]
             arg = iarg and data[int(iarg)]
+            if iextra:
+                alphabet = data[int(iextra)]()
+                func = lambda t, s: data[int(ifunc)](t, s, alphabet)
+            else:
+                func = data[int(ifunc)]
+
             # Uncomment to debug transformation chain
             # nprev = ''.join(n)
             # dprev = ' '.join(map(prd, data))

From 4d83c072b195f4deaf41f0ac08b909dfc87fa13a Mon Sep 17 00:00:00 2001
From: Conner <65339198+connercsbn@users.noreply.github.com>
Date: Wed, 17 Nov 2021 14:41:34 -0800
Subject: [PATCH 050/394] drop unnecessary compat_str

---
 youtube_dl/extractor/podchaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/podchaser.py b/youtube_dl/extractor/podchaser.py
index 80f970c1b2b..a6b9752fe4d 100644
--- a/youtube_dl/extractor/podchaser.py
+++ b/youtube_dl/extractor/podchaser.py
@@ -59,8 +59,8 @@ class PodchaserIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        audio_id, podcast_id, creator_id = compat_str(mobj.group('id')), \
-            compat_str(mobj.group('podcast_id')), compat_str(mobj.group('creator_id'))
+        audio_id, podcast_id, creator_id = mobj.group('id'), \
+            mobj.group('podcast_id'), mobj.group('creator_id')
         webpage = self._download_webpage(url, audio_id)
         page_title = self._html_search_meta(['title', 'og:title', 'twitter:title'], webpage, default=None) \
             or self._search_regex(

From 89ef8b9b99a5cdf24a81dc3070106302bef64724 Mon Sep 17 00:00:00 2001
From: Conner <65339198+connercsbn@users.noreply.github.com>
Date: Wed, 17 Nov 2021 14:43:21 -0800
Subject: [PATCH 051/394] regex tweak

---
 youtube_dl/extractor/podchaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/podchaser.py b/youtube_dl/extractor/podchaser.py
index a6b9752fe4d..95509764e72 100644
--- a/youtube_dl/extractor/podchaser.py
+++ b/youtube_dl/extractor/podchaser.py
@@ -11,7 +11,7 @@ class PodchaserIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?podchaser\.com/
         (?:(?:podcasts)|(?:creators))
-        /[\w\-]+-
+        /[\w-]+-
         (?:
             (?P<creator_id>[\d]+[\w]+)|(?P<podcast_id>[\d]+))
         (?:/episodes/[\w\-]+-

From 820999dd2c28ad0b823911a8d0085093c31a205b Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 052/394] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index da73e130c2f..c745b121f3d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From 622b87d0fb10a50283b12ecd5304e66dd396809b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 18 Nov 2021 01:42:07 +0000
Subject: [PATCH 053/394] Update for player 68e11abe

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c5e3a300fe9..da73e130c2f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1543,7 +1543,8 @@ def remove(s, i):
             ('alphabet2', lambda: b64_alphabets[1],
                           # noqa: E127
                           # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
-                          r"^[^}]+?case\s58:d=44;[^}]+\}return",
+                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 58:d-=14;case 91:case 92:case 93:continue;case 123:d=47;case 94:case 95:case 96:continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
+                          r"^[^}]+?case\s58:d(?:=44;|-=14;[^}]+\})[^}]+\}return",
                           def_skip),
             ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
                          # noqa: E127

From 011e1bef45723d23d8a2cfddd41e5adc55d2ced4 Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Mon, 22 Nov 2021 19:32:33 +0400
Subject: [PATCH 054/394] Updated to include Cloudfront url (Quick Fix)

---
 youtube_dl/extractor/streamff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
index 83ab398fc27..8bcecdad6b5 100644
--- a/youtube_dl/extractor/streamff.py
+++ b/youtube_dl/extractor/streamff.py
@@ -22,5 +22,5 @@ class StreamFFIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_url = "https://streamff.streamff.com/uploads/%s.mp4" % video_id
+        video_url = "https://d32h9bi2u31gge.cloudfront.net/uploads/%s.mp4" % video_id
         return self.url_result(video_url, ie="Generic")

From 6ddf44e2f31dc74d819cfe9fbc25b717e214e40e Mon Sep 17 00:00:00 2001
From: Conner <65339198+connercsbn@users.noreply.github.com>
Date: Mon, 22 Nov 2021 17:39:26 -0800
Subject: [PATCH 055/394] use unified_timestamp

---
 youtube_dl/extractor/podchaser.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/podchaser.py b/youtube_dl/extractor/podchaser.py
index 95509764e72..b3f80ab2aef 100644
--- a/youtube_dl/extractor/podchaser.py
+++ b/youtube_dl/extractor/podchaser.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 import re
 from datetime import datetime
-from ..utils import float_or_none, try_get, str_to_int
+from ..utils import float_or_none, try_get, str_to_int, unified_timestamp
 from ..compat import compat_str
 from .common import InfoExtractor
 
@@ -28,7 +28,7 @@ class PodchaserIE(InfoExtractor):
             'categories': ['Comedy'],
             'series': 'Cum Town',
             'duration': 3708,
-            'timestamp': 1636560059,
+            'timestamp': 1636531259,
             'upload_date': '20211110'
         }
     }, {
@@ -44,7 +44,7 @@ class PodchaserIE(InfoExtractor):
         'info_dict': {
             'id': '107ZzkFiEQ',
             'title': 'Todd Glass',
-            'description': 'md5:f0302588c98d474d4e8551f4914042c9',
+            'description': 'md5:0771e81d879f304f11254e5a56a97a58',
         },
         'playlist_mincount': 48
     }, {
@@ -52,7 +52,7 @@ class PodchaserIE(InfoExtractor):
         'info_dict': {
             'id': '699349',
             'title': "Sean Carroll's Mindscape: Science, Society, Philosophy, Culture, Arts, and Ideas",
-            'description': 'md5:a1ed5db77378a736b3a85a2331cbc05a'
+            'description': 'md5:8692ce0c50cb900c5e4eb27b437dd67b'
         },
         'playlist_count': 25
     }]
@@ -83,7 +83,7 @@ def _real_extract(self, url):
             'url': episode.get('audio_url'),
             'thumbnail': try_get(episode, lambda x: x['podcast']['image_url']),
             'duration': str_to_int(episode.get('length')),
-            'timestamp': datetime.strptime(episode.get('air_date'), '%Y-%m-%d %H:%M:%S').timestamp(),
+            'timestamp': unified_timestamp(episode.get('air_date')),
             'rating': float_or_none(episode.get('rating')),
             'categories': [x['text'] for x in try_get(episode, lambda x: x['podcast']['categories'], list) or []],
             'tags': [tag['text'] for tag in episode.get('tags') or []],

From 1b96fd20f1f6290578ff40871036a7ed2b00173d Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 056/394] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e61157bfafc..bc7932f8c62 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From d69983a8756fa75e16606bcd1b2ba50ec115ed16 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 23 Nov 2021 18:53:35 +0000
Subject: [PATCH 057/394] Update for player f1ca6900

Numbers can be float literals, eg '1E3'
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index da73e130c2f..e61157bfafc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1589,9 +1589,9 @@ def remove(s, i):
             elif re.match(r'^"[^"]*",', datac):
                 el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
             # Integer input data
-            elif re.match(r'^-?\d+,', datac):
+            elif re.match(r'^-?\d+(?:E\d+)?,', datac):
                 el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
-                el = int(el)
+                el = int(float(el) if 'E' in el else el)
             # Reference to "n" parameter array
             elif re.match('^b,', datac):
                 el = n

From a54d9d1c1d3d55fcea6b4e45bf510d29817faf0d Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Tue, 23 Nov 2021 22:34:06 +0100
Subject: [PATCH 058/394] update version

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 509aeb19ef1..1f12ba4fc7e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.11.13'
+__version__ = '2021.11.23'

From 9cc50f832028f8bbf35f2019cc7442bcdc4345ed Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Tue, 23 Nov 2021 22:36:59 +0100
Subject: [PATCH 059/394] cleaning

---
 docs/supportedsites.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ae2a6b8b02b..8bc14d81dce 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -717,6 +717,7 @@
  - **Playwire**
  - **pluralsight**
  - **pluralsight:course**
+ - **Podchaser**
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
@@ -900,6 +901,7 @@
  - **Streamable**
  - **streamcloud.eu**
  - **StreamCZ**
+ - **StreamFF**
  - **StreetVoice**
  - **StretchInternet**
  - **stv:player**

From 97f1416fe3e6003c19a14c7c108dd3f41537eb7c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 24 Nov 2021 15:41:22 +0000
Subject: [PATCH 060/394] [itv] fixed extraction (closes #28906)

ITV changes require media stream and subtitles to be taken
from different playlist resources.

Original commit 4c57dd2 by @sleaux-meaux 3 May 2021
---
 youtube_dl/extractor/itv.py | 102 ++++++++++++++++++++++++++++--------
 1 file changed, 80 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index e86c40b42e5..55b9cfc1f0e 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -10,11 +10,13 @@
     clean_html,
     determine_ext,
     extract_attributes,
+    ExtractorError,
     get_element_by_class,
     JSON_LD_RE,
     merge_dicts,
     parse_duration,
     smuggle_url,
+    try_get,
     url_or_none,
 )
 
@@ -23,15 +25,15 @@ class ITVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
     _GEO_COUNTRIES = ['GB']
     _TESTS = [{
-        'url': 'https://www.itv.com/hub/liar/2a4547a0012',
+        'url': 'https://www.itv.com/hub/plebs/2a1873a0002',
         'info_dict': {
-            'id': '2a4547a0012',
+            'id': '2a1873a0002',
             'ext': 'mp4',
-            'title': 'Liar - Series 2 - Episode 6',
-            'description': 'md5:d0f91536569dec79ea184f0a44cca089',
-            'series': 'Liar',
-            'season_number': 2,
-            'episode_number': 6,
+            'title': 'Plebs - The Orgy',
+            'description': 'md5:4d7159af53ebd5b36e8b3ec82a41fdb4',
+            'series': 'Plebs',
+            'season_number': 1,
+            'episode_number': 1
         },
         'params': {
             # m3u8 download
@@ -56,6 +58,22 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, video_id)
         params = extract_attributes(self._search_regex(
             r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
+        variants = self._parse_json(
+            try_get(params, lambda x: x['data-video-variants'], str) or '{}',
+            video_id, fatal=False)
+        platform_tag_video = None
+        featureset_video = None
+        for platform_tag, featuresets in variants.items():
+            for featureset in featuresets:
+                if (isinstance(featureset, list) and len(featureset) >= 2
+                        and featureset[0] == 'hls' and featureset[1] == 'aes'):
+                    platform_tag_video = platform_tag
+                    featureset_video = featureset
+                    break
+        if not platform_tag_video or not featureset_video:
+            raise ExtractorError(
+                '[%s] %s: %s' % (self.IE_NAME, video_id, 'No downloads available'),
+                expected=True)
 
         ios_playlist_url = params.get('data-video-playlist') or params['data-video-id']
         hmac = params['data-video-hmac']
@@ -87,13 +105,13 @@ def _real_extract(self, url):
                 },
                 'variantAvailability': {
                     'featureset': {
-                        'min': ['hls', 'aes', 'outband-webvtt'],
-                        'max': ['hls', 'aes', 'outband-webvtt']
+                        'min': featureset_video,
+                        'max': featureset_video
                     },
-                    'platformTag': 'dotcom'
+                    'platformTag': platform_tag_video
                 }
             }).encode(), headers=headers)
-        video_data = ios_playlist['Playlist']['Video']
+        video_data = try_get(ios_playlist, lambda x: x['Playlist']['Video'], dict) or {}
         ios_base_url = video_data.get('Base')
 
         formats = []
@@ -115,17 +133,57 @@ def _real_extract(self, url):
         self._sort_formats(formats)
 
         subtitles = {}
-        subs = video_data.get('Subtitles') or []
-        for sub in subs:
-            if not isinstance(sub, dict):
-                continue
-            href = url_or_none(sub.get('Href'))
-            if not href:
-                continue
-            subtitles.setdefault('en', []).append({
-                'url': href,
-                'ext': determine_ext(href, 'vtt'),
-            })
+        platform_tag_subs = None
+        featureset_subs = None
+        for platform_tag, featuresets in variants.items():
+            for featureset in featuresets:
+                if (isinstance(featureset, list) and len(featureset) >= 3
+                        and featureset[2] == 'outband-webvtt'):
+                    platform_tag_subs = platform_tag
+                    featureset_subs = featureset
+                    break
+        if not platform_tag_subs or not featureset_subs:
+            self.report_warning('%s: %s' % (video_id, 'No subtitles available'))
+        else:
+            subs_playlist = self._download_json(
+                ios_playlist_url, video_id, data=json.dumps({
+                    'user': {
+                        'itvUserId': '',
+                        'entitlements': [],
+                        'token': ''
+                    },
+                    'device': {
+                        'manufacturer': 'Safari',
+                        'model': '5',
+                        'os': {
+                            'name': 'Windows NT',
+                            'version': '6.1',
+                            'type': 'desktop'
+                        }
+                    },
+                    'client': {
+                        'version': '4.1',
+                        'id': 'browser'
+                    },
+                    'variantAvailability': {
+                        'featureset': {
+                            'min': featureset_subs,
+                            'max': featureset_subs
+                        },
+                        'platformTag': platform_tag_subs
+                    }
+                }).encode(), headers=headers)
+            subs = try_get(subs_playlist, lambda x: x['Playlist']['Video']['Subtitles'], list) or []
+            for sub in subs:
+                if not isinstance(sub, dict):
+                    continue
+                href = url_or_none(sub.get('Href'))
+                if not href:
+                    continue
+                subtitles.setdefault('en', []).append({
+                    'url': href,
+                    'ext': determine_ext(href, 'vtt'),
+                })
 
         info = self._search_json_ld(webpage, video_id, default={})
         if not info:

From a72aca56673fdcfb45e23b8df8434fb5361d983e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 24 Nov 2021 18:56:28 +0000
Subject: [PATCH 061/394] Back-port ITV extractor from yt-dlp PR #1776

Includes commit 6fbcc16, fix by staubichsauger & pukkandan
---
 youtube_dl/extractor/itv.py | 236 ++++++++++++++++++++----------------
 1 file changed, 132 insertions(+), 104 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 55b9cfc1f0e..4acfa81ad60 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -2,11 +2,13 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveNewIE
+
+from ..compat import compat_str
 from ..utils import (
+    base_url,
     clean_html,
     determine_ext,
     extract_attributes,
@@ -18,6 +20,8 @@
     smuggle_url,
     try_get,
     url_or_none,
+    url_basename,
+    urljoin,
 )
 
 
@@ -33,7 +37,24 @@ class ITVIE(InfoExtractor):
             'description': 'md5:4d7159af53ebd5b36e8b3ec82a41fdb4',
             'series': 'Plebs',
             'season_number': 1,
-            'episode_number': 1
+            'episode_number': 1,
+            'thumbnail': r're:https?://hubimages\.itv\.com/episode/2_1873_0002'
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.itv.com/hub/the-jonathan-ross-show/2a1166a0209',
+        'info_dict': {
+            'id': '2a1166a0209',
+            'ext': 'mp4',
+            'title': 'The Jonathan Ross Show - Series 17 - Episode 8',
+            'description': 'md5:3023dcdd375db1bc9967186cdb3f1399',
+            'series': 'The Jonathan Ross Show',
+            'episode_number': 8,
+            'season_number': 17,
+            'thumbnail': r're:https?://hubimages\.itv\.com/episode/2(?:_\d{4}){2}'
         },
         'params': {
             # m3u8 download
@@ -53,38 +74,16 @@ class ITVIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        params = extract_attributes(self._search_regex(
-            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
-        variants = self._parse_json(
-            try_get(params, lambda x: x['data-video-variants'], str) or '{}',
-            video_id, fatal=False)
-        platform_tag_video = None
-        featureset_video = None
-        for platform_tag, featuresets in variants.items():
-            for featureset in featuresets:
-                if (isinstance(featureset, list) and len(featureset) >= 2
-                        and featureset[0] == 'hls' and featureset[1] == 'aes'):
-                    platform_tag_video = platform_tag
-                    featureset_video = featureset
-                    break
-        if not platform_tag_video or not featureset_video:
-            raise ExtractorError(
-                '[%s] %s: %s' % (self.IE_NAME, video_id, 'No downloads available'),
-                expected=True)
-
-        ios_playlist_url = params.get('data-video-playlist') or params['data-video-id']
-        hmac = params['data-video-hmac']
-        headers = self.geo_verification_headers()
-        headers.update({
+    def _generate_api_headers(self, hmac):
+        return merge_dicts({
             'Accept': 'application/vnd.itv.vod.playlist.v2+json',
             'Content-Type': 'application/json',
             'hmac': hmac.upper(),
-        })
-        ios_playlist = self._download_json(
-            ios_playlist_url, video_id, data=json.dumps({
+        }, self.geo_verification_headers())
+
+    def _call_api(self, video_id, playlist_url, headers, platform_tag, featureset, fatal=True):
+        return self._download_json(
+            playlist_url, video_id, data=json.dumps({
                 'user': {
                     'itvUserId': '',
                     'entitlements': [],
@@ -105,15 +104,62 @@ def _real_extract(self, url):
                 },
                 'variantAvailability': {
                     'featureset': {
-                        'min': featureset_video,
-                        'max': featureset_video
+                        'min': featureset,
+                        'max': featureset
                     },
-                    'platformTag': platform_tag_video
+                    'platformTag': platform_tag
                 }
-            }).encode(), headers=headers)
+            }).encode(), headers=headers, fatal=fatal)
+
+    def _get_subtitles(self, video_id, variants, ios_playlist_url, headers, *args, **kwargs):
+        subtitles = {}
+        # Prefer last matching featureset
+        # See: https://github.com/yt-dlp/yt-dlp/issues/986
+        platform_tag_subs, featureset_subs = next(
+            ((platform_tag, featureset)
+             for platform_tag, featuresets in reversed(tuple(variants.items())) for featureset in featuresets
+             if try_get(featureset, lambda x: x[2]) == 'outband-webvtt'),
+            (None, None))
+
+        if platform_tag_subs and featureset_subs:
+            subs_playlist = self._call_api(
+                video_id, ios_playlist_url, headers, platform_tag_subs, featureset_subs, fatal=False)
+            subs = try_get(subs_playlist, lambda x: x['Playlist']['Video']['Subtitles'], list) or []
+            for sub in subs:
+                if not isinstance(sub, dict):
+                    continue
+                href = url_or_none(sub.get('Href'))
+                if not href:
+                    continue
+                subtitles.setdefault('en', []).append({'url': href})
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        params = extract_attributes(self._search_regex(
+            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
+        variants = self._parse_json(
+            try_get(params, lambda x: x['data-video-variants'], compat_str) or '{}',
+            video_id, fatal=False)
+        # Prefer last matching featureset
+        # See: https://github.com/yt-dlp/yt-dlp/issues/986
+        platform_tag_video, featureset_video = next(
+            ((platform_tag, featureset)
+             for platform_tag, featuresets in reversed(tuple(variants.items())) for featureset in featuresets
+             if set(try_get(featureset, lambda x: x[:2]) or {}) == {'aes', 'hls'}),
+            (None, None))
+        if not platform_tag_video or not featureset_video:
+            raise ExtractorError('No downloads available', expected=True, video_id=video_id)
+
+        ios_playlist_url = params.get('data-video-playlist') or params['data-video-id']
+        headers = self._generate_api_headers(params['data-video-hmac'])
+        headers['Referer'] = url
+        ios_playlist = self._call_api(
+            video_id, ios_playlist_url, headers, platform_tag_video, featureset_video)
+
         video_data = try_get(ios_playlist, lambda x: x['Playlist']['Video'], dict) or {}
         ios_base_url = video_data.get('Base')
-
         formats = []
         for media_file in (video_data.get('MediaFiles') or []):
             href = media_file.get('Href')
@@ -131,60 +177,6 @@ def _real_extract(self, url):
                     'url': href,
                 })
         self._sort_formats(formats)
-
-        subtitles = {}
-        platform_tag_subs = None
-        featureset_subs = None
-        for platform_tag, featuresets in variants.items():
-            for featureset in featuresets:
-                if (isinstance(featureset, list) and len(featureset) >= 3
-                        and featureset[2] == 'outband-webvtt'):
-                    platform_tag_subs = platform_tag
-                    featureset_subs = featureset
-                    break
-        if not platform_tag_subs or not featureset_subs:
-            self.report_warning('%s: %s' % (video_id, 'No subtitles available'))
-        else:
-            subs_playlist = self._download_json(
-                ios_playlist_url, video_id, data=json.dumps({
-                    'user': {
-                        'itvUserId': '',
-                        'entitlements': [],
-                        'token': ''
-                    },
-                    'device': {
-                        'manufacturer': 'Safari',
-                        'model': '5',
-                        'os': {
-                            'name': 'Windows NT',
-                            'version': '6.1',
-                            'type': 'desktop'
-                        }
-                    },
-                    'client': {
-                        'version': '4.1',
-                        'id': 'browser'
-                    },
-                    'variantAvailability': {
-                        'featureset': {
-                            'min': featureset_subs,
-                            'max': featureset_subs
-                        },
-                        'platformTag': platform_tag_subs
-                    }
-                }).encode(), headers=headers)
-            subs = try_get(subs_playlist, lambda x: x['Playlist']['Video']['Subtitles'], list) or []
-            for sub in subs:
-                if not isinstance(sub, dict):
-                    continue
-                href = url_or_none(sub.get('Href'))
-                if not href:
-                    continue
-                subtitles.setdefault('en', []).append({
-                    'url': href,
-                    'ext': determine_ext(href, 'vtt'),
-                })
-
         info = self._search_json_ld(webpage, video_id, default={})
         if not info:
             json_ld = self._parse_json(self._search_regex(
@@ -198,36 +190,73 @@ def _real_extract(self, url):
                         info = self._json_ld(item, video_id, fatal=False) or {}
                         break
 
+        thumbnails = []
+        thumbnail_url = try_get(params, lambda x: x['data-video-posterframe'], compat_str)
+        if thumbnail_url:
+            thumbnails.extend([{
+                'url': thumbnail_url.format(width=1920, height=1080, quality=100, blur=0, bg='false'),
+                'width': 1920,
+                'height': 1080,
+            }, {
+                'url': urljoin(base_url(thumbnail_url), url_basename(thumbnail_url)),
+                'preference': -2
+            }])
+
+        thumbnail_url = self._html_search_meta(['og:image', 'twitter:image'], webpage, default=None)
+        if thumbnail_url:
+            thumbnails.append({
+                'url': thumbnail_url,
+            })
+        self._remove_duplicate_formats(thumbnails)
+
         return merge_dicts({
             'id': video_id,
             'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
             'formats': formats,
-            'subtitles': subtitles,
+            'subtitles': self.extract_subtitles(video_id, variants, ios_playlist_url, headers),
             'duration': parse_duration(video_data.get('Duration')),
             'description': clean_html(get_element_by_class('episode-info__synopsis', webpage)),
+            'thumbnails': thumbnails
         }, info)
 
 
 class ITVBTCCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:news|btcc)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action',
+        'info_dict': {
+            'id': 'btcc-2019-brands-hatch-gp-race-action',
+            'title': 'BTCC 2019: Brands Hatch GP race action',
+        },
+        'playlist_count': 12,
+    }, {
+        'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
         'info_dict': {
-            'id': 'btcc-2018-all-the-action-from-brands-hatch',
-            'title': 'BTCC 2018: All the action from Brands Hatch',
+            'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+            'title': 'md5:6ef054dd9f069330db3dcc66cb772d32'
         },
-        'playlist_mincount': 9,
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
+        'playlist_count': 4
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
-        entries = [
-            self.url_result(
-                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
+        json_map = try_get(self._parse_json(self._html_search_regex(
+            '(?s)<script[^>]+id=[\'"]__NEXT_DATA__[^>]*>([^<]+)</script>', webpage, 'json_map'), playlist_id),
+            lambda x: x['props']['pageProps']['article']['body']['content']) or []
+
+        entries = []
+        for video in json_map:
+            if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')):
+                continue
+            video_id = video['data']['id']
+            account_id = video['data']['accountId']
+            player_id = video['data']['playerId']
+            entries.append(self.url_result(
+                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id), {
                     # ITV does not like some GB IP ranges, so here are some
                     # IP blocks it accepts
                     'geo_ip_blocks': [
@@ -235,8 +264,7 @@ def _real_extract(self, url):
                     ],
                     'referrer': url,
                 }),
-                ie=BrightcoveNewIE.ie_key(), video_id=video_id)
-            for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]
+                ie=BrightcoveNewIE.ie_key(), video_id=video_id))
 
         title = self._og_search_title(webpage, fatal=False)
 

From d210e1862fb43ed571cf421e04f8748cc1448944 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Fri, 26 Nov 2021 10:39:17 +0900
Subject: [PATCH 062/394] [YoutubeDL] modify fixup warning messages

---
 youtube_dl/YoutubeDL.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef9c..97bad976b4d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,8 +1994,8 @@ def compatible_formats(formats):
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
-                            info_dict['id'], stretched_ratio))
+                        self.report_warning('Non-uniform pixel ratio (%f) in "%s".' % (
+                            stretched_ratio, filename))
                     elif fixup_policy == 'detect_or_warn':
                         stretched_pp = FFmpegFixupStretchedPP(self)
                         if stretched_pp.available:
@@ -2003,8 +2003,8 @@ def compatible_formats(formats):
                             info_dict['__postprocessors'].append(stretched_pp)
                         else:
                             self.report_warning(
-                                '%s: Non-uniform pixel ratio (%s). %s'
-                                % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
+                                'Non-uniform pixel ratio (%f) in "%s". %s'
+                                % (stretched_ratio, filename, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
@@ -2012,9 +2012,9 @@ def compatible_formats(formats):
                         and info_dict.get('container') == 'm4a_dash'):
                     if fixup_policy == 'warn':
                         self.report_warning(
-                            '%s: writing DASH m4a. '
+                            'Writing DASH m4a in "%s". '
                             'Only some players support this container.'
-                            % info_dict['id'])
+                            % filename)
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM4aPP(self)
                         if fixup_pp.available:
@@ -2022,9 +2022,9 @@ def compatible_formats(formats):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: writing DASH m4a. '
+                                'Writing DASH m4a in "%s". '
                                 'Only some players support this container. %s'
-                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
+                                % (filename, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 

From 5bca20c0e7481a98cd96493cb195f2784093a94f Mon Sep 17 00:00:00 2001
From: Chris Rose <offline@offby1.net>
Date: Fri, 26 Nov 2021 08:08:17 -0800
Subject: [PATCH 063/394] xvideos: Fix for #30271

---
 youtube_dl/extractor/xvideos.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 8fc64914caf..e63d4690d4b 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -82,7 +82,7 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.xvideos.com/video%s/' % video_id, video_id)
+            'https://www.xvideos.com/video%s/0' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:

From 4ca269b7dbce9174769f91e26674a6c82195b737 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 4 Nov 2021 12:48:06 +0000
Subject: [PATCH 064/394] Handle default in switch better

Add https://github.com/yt-dlp/yt-dlp/commit/a1fc7ca0743c8df06416e68ee74b64e07dfe7135
Thanks coletdjnz
---
 test/test_jsinterp.py  | 15 +++++++++++++++
 youtube_dl/jsinterp.py | 23 ++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d05ea61013..acdabffb1f8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -133,6 +133,21 @@ def test_switch(self):
         self.assertEqual(jsi.call_function('x', 3), 6)
         self.assertEqual(jsi.call_function('x', 5), 0)
 
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
     def test_try(self):
         jsi = JSInterpreter('''
         function x() { try{return 10} catch(e){return 5} }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 061e92c2a4a..c35765702a6 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,21 +240,26 @@ def raise_constructor_error(c):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
-            body, default = body.split('default:') if 'default:' in body else (body, None)
-            items = body.split('case ')[1:]
-            if default:
-                items.append('default:%s' % (default, ))
-            matched = False
-            for item in items:
-                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
-                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
-                if matched:
+            items = body.replace('default:', 'case default:').split('case ')[1:]
+            for default in (False, True):
+                matched = False
+                for item in items:
+                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    if default:
+                        matched = matched or case == 'default'
+                    elif not matched:
+                        matched = (case != 'default'
+                                   and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
+                    if not matched:
+                        continue
                     try:
                         ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
+                if matched:
+                    break
             return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
 
         # Comma separated statements

From cabfef78844f371cdd9fe417179d0955e3a7bf77 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 02:06:13 +0000
Subject: [PATCH 065/394] Fix splice to handle float

Needed for new youtube js player f1ca6900
Add https://github.com/yt-dlp/yt-dlp/commit/57dbe8077f8d00e0fffac53669f40cd7d584474f#diff-729b57caa8d006426f6a8960c061f519a8b6658682284015e069745af52ffb07
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c35765702a6..c75cf45b954 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -416,7 +416,7 @@ def eval_method():
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')
-                    index, howMany = (argvals + [len(obj)])[:2]
+                    index, howMany = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
                     add_items = argvals[2:]

From 617f065d1574d26cbcbb727fea66caebf58a8c89 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 03:18:29 +0000
Subject: [PATCH 066/394] Back-port test_youtube_signature.py from yt-dlp and
 fix JSInterp accordingly

---
 test/test_youtube_signature.py | 87 ++++++++++++++++++++++++----------
 youtube_dl/jsinterp.py         |  9 ++--
 2 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 627d4cb9255..c8e85b5005a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -14,9 +14,10 @@
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.jsinterp import JSInterpreter
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
@@ -95,35 +115,54 @@ def setUp(self):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
-    assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+def t_factory(name, sig_func, url_pattern):
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
+
+        def test_func(self):
+            basename = 'player-{0}-{1}.js'.format(name, test_id)
+            fn = os.path.join(self.TESTDATA_DIR, basename)
+
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
+
+        test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
+
 
-    def test_func(self):
-        basename = 'player-%s.js' % test_id
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
 
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
-        with io.open(fn, encoding='utf-8') as testf:
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
+def n_sig(jscode, sig_input):
+    # Pending implementation of _extract_n_function_name() or similar in
+    # youtube.py, hard-code here
+    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    import re
+    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
+    funcname = funcname and funcname.group(1)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
 
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
 
-for test_spec in _TESTS:
-    make_tfunc(*test_spec)
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c75cf45b954..a2306557b4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -9,7 +9,8 @@
     remove_quotes,
 )
 from .compat import (
-    compat_collections_abc
+    compat_collections_abc,
+    compat_str,
 )
 MutableMapping = compat_collections_abc.MutableMapping
 
@@ -372,7 +373,7 @@ def eval_method():
                 # nonlocal member
                 member = nl.member
                 if variable == 'String':
-                    obj = str
+                    obj = compat_str
                 elif variable in local_vars:
                     obj = local_vars[variable]
                 else:
@@ -391,7 +392,7 @@ def eval_method():
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == str:
+                if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
@@ -533,7 +534,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    [x.strip() for x in mobj.group('args').split(',')],
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)

From 10d6f726b64f495ec4b36c9d4186b25632e81c7c Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 29 Nov 2021 12:27:48 +0100
Subject: [PATCH 067/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1f12ba4fc7e..2c039b2a964 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.11.23'
+__version__ = '2021.11.29'

From 6cf8f2e09fbcc27ebf7066e7d3d4966db2d0e874 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 18 Apr 2021 01:46:40 +0100
Subject: [PATCH 068/394] Try for timestamp, description from
 window.__INITIAL_DATA__ pages

---
 youtube_dl/extractor/bbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 247d982ce65..37d427a66c8 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1205,7 +1205,10 @@ def parse_media(media):
                 if name == 'media-experience':
                     parse_media(try_get(resp, lambda x: x['data']['initialItem']['mediaItem'], dict))
                 elif name == 'article':
-                    for block in (try_get(resp, lambda x: x['data']['blocks'], list) or []):
+                    for block in (try_get(resp,
+                                          (lambda x: x['data']['blocks'],
+                                           lambda x: x['data']['content']['model']['blocks'],),
+                                          list) or []):
                         if block.get('type') != 'media':
                             continue
                         parse_media(block.get('model'))

From 70627b502e5260dc22cdb441f1c89734323e8a33 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 30 Nov 2021 05:15:33 +0000
Subject: [PATCH 069/394] Support __INITIAL_DATA__ with stringified JSON

Add test and fix test for bbcthreeConfig
---
 youtube_dl/extractor/bbc.py | 50 +++++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 37d427a66c8..088af982323 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -12,6 +12,7 @@
     compat_HTTPError,
     compat_parse_qs,
     compat_str,
+    compat_urllib_error,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
@@ -395,9 +396,17 @@ def _process_media_selector(self, media_selection, programme_id):
                         formats.extend(self._extract_mpd_formats(
                             href, programme_id, mpd_id=format_id, fatal=False))
                     elif transfer_format == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
-                            href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                            m3u8_id=format_id, fatal=False))
+                        # TODO: let expected_status be passed into _extract_xxx_formats() instead
+                        try:
+                            fmts = self._extract_m3u8_formats(
+                                href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                                m3u8_id=format_id, fatal=False)
+                        except ExtractorError as e:
+                            if not (isinstance(e.exc_info[1], compat_urllib_error.HTTPError)
+                                    and e.exc_info[1].code in (403, 404)):
+                                raise
+                            fmts = []
+                        formats.extend(fmts)
                     elif transfer_format == 'hds':
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))
@@ -775,21 +784,33 @@ class BBCIE(BBCCoUkIE):
             'timestamp': 1437785037,
             'upload_date': '20150725',
         },
+    }, {
+        # video with window.__INITIAL_DATA__ and value as JSON string
+        'url': 'https://www.bbc.com/news/av/world-europe-59468682',
+        'info_dict': {
+            'id': 'p0b71qth',
+            'ext': 'mp4',
+            'title': 'Why France is making this woman a national hero',
+            'description': 'md5:7affdfab80e9c3a1f976230a1ff4d5e4',
+            'thumbnail': r're:https?://.+/.+\.jpg',
+            'timestamp': 1638230731,
+            'upload_date': '20211130',
+        },
     }, {
         # single video article embedded with data-media-vpid
         'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
         'only_matching': True,
     }, {
+        # bbcthreeConfig
         'url': 'https://www.bbc.co.uk/bbcthree/clip/73d0bbd0-abc3-4cea-b3c0-cdae21905eb1',
         'info_dict': {
             'id': 'p06556y7',
             'ext': 'mp4',
-            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
-            'description': 'md5:4b7dfd063d5a789a1512e99662be3ddd',
+            'title': 'Things Not To Say to people that live on council estates',
+            'description': "From being labelled a 'chav', to the presumption that they're 'scroungers', people who live on council estates encounter all kinds of prejudices and false assumptions about themselves, their families, and their lifestyles. Here, eight people discuss the common statements, misconceptions, and clichés that they're tired of hearing.",
+            'duration': 360,
+            'thumbnail': r're:https?://.+/.+\.jpg',
         },
-        'params': {
-            'skip_download': True,
-        }
     }, {
         # window.__PRELOADED_STATE__
         'url': 'https://www.bbc.co.uk/radio/play/b0b9z4yl',
@@ -1162,9 +1183,16 @@ def _real_extract(self, url):
                 return self.playlist_result(
                     entries, playlist_id, playlist_title, playlist_description)
 
-        initial_data = self._parse_json(self._search_regex(
-            r'window\.__INITIAL_DATA__\s*=\s*({.+?});', webpage,
-            'preload state', default='{}'), playlist_id, fatal=False)
+        initial_data = self._search_regex(
+            r'window\.__INITIAL_DATA__\s*=\s*("{.+?}")\s*;', webpage,
+            'quoted preload state', default=None)
+        if initial_data is None:
+            initial_data = self._search_regex(
+                r'window\.__INITIAL_DATA__\s*=\s*({.+?})\s*;', webpage,
+                'preload state', default={})
+        else:
+            initial_data = self._parse_json(initial_data or '"{}"', playlist_id, fatal=False)
+        initial_data = self._parse_json(initial_data, playlist_id, fatal=False)
         if initial_data:
             def parse_media(media):
                 if not media:

From 22d6a98d1f108be9fb39befb70918b5dc278f01b Mon Sep 17 00:00:00 2001
From: Guy Porat <56124817+ItsKlork@users.noreply.github.com>
Date: Tue, 30 Nov 2021 14:46:43 +0200
Subject: [PATCH 070/394] Strip path after sanitizing

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58de..d5f5eddc339 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2129,7 +2129,7 @@ def sanitize_path(s):
     if drive_or_unc:
         norm_path.pop(0)
     sanitized_path = [
-        path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
+        path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part).strip()
         for path_part in norm_path]
     if drive_or_unc:
         sanitized_path.insert(0, drive_or_unc + os.path.sep)

From bb69eb13bf339e0417e5775951b9198fc0759a90 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Wed, 1 Dec 2021 11:48:46 +0900
Subject: [PATCH 071/394] revert DASH m4a message modification

---
 youtube_dl/YoutubeDL.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 97bad976b4d..030629ed090 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2012,9 +2012,9 @@ def compatible_formats(formats):
                         and info_dict.get('container') == 'm4a_dash'):
                     if fixup_policy == 'warn':
                         self.report_warning(
-                            'Writing DASH m4a in "%s". '
+                            '%s: writing DASH m4a. '
                             'Only some players support this container.'
-                            % filename)
+                            % info_dict['id'])
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM4aPP(self)
                         if fixup_pp.available:
@@ -2022,9 +2022,9 @@ def compatible_formats(formats):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                'Writing DASH m4a in "%s". '
+                                '%s: writing DASH m4a. '
                                 'Only some players support this container. %s'
-                                % (filename, INSTALL_FFMPEG_MESSAGE))
+                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 

From 52774c391a040925f3c3f8198d2e2911aa4185f3 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Wed, 1 Dec 2021 12:02:09 +0900
Subject: [PATCH 072/394] [YoutubeDL] modify fixup warning message

---
 youtube_dl/YoutubeDL.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef9c..030629ed090 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,8 +1994,8 @@ def compatible_formats(formats):
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
-                            info_dict['id'], stretched_ratio))
+                        self.report_warning('Non-uniform pixel ratio (%f) in "%s".' % (
+                            stretched_ratio, filename))
                     elif fixup_policy == 'detect_or_warn':
                         stretched_pp = FFmpegFixupStretchedPP(self)
                         if stretched_pp.available:
@@ -2003,8 +2003,8 @@ def compatible_formats(formats):
                             info_dict['__postprocessors'].append(stretched_pp)
                         else:
                             self.report_warning(
-                                '%s: Non-uniform pixel ratio (%s). %s'
-                                % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
+                                'Non-uniform pixel ratio (%f) in "%s". %s'
+                                % (stretched_ratio, filename, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 

From 5adea4baa3d540cdbe6f4dffd3df515d63d83e93 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Wed, 1 Dec 2021 14:06:35 +0900
Subject: [PATCH 073/394] modify post-processing for MPEG-DASH

---
 youtube_dl/YoutubeDL.py              | 16 +++++++++-------
 youtube_dl/postprocessor/__init__.py |  4 ++--
 youtube_dl/postprocessor/ffmpeg.py   | 10 ++++++++--
 3 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef9c..28d18b13e23 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -101,7 +101,7 @@
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
     FFmpegFixupM3u8PP,
-    FFmpegFixupM4aPP,
+    FFmpegFixupMpegDashPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
@@ -2009,22 +2009,24 @@ def compatible_formats(formats):
                         assert fixup_policy in ('ignore', 'never')
 
                 if (info_dict.get('requested_formats') is None
-                        and info_dict.get('container') == 'm4a_dash'):
+                        and info_dict.get('container') in ('m4a_dash', 'mp4_dash')):
                     if fixup_policy == 'warn':
                         self.report_warning(
-                            '%s: writing DASH m4a. '
+                            'Writing DASH %s in "%s". '
                             'Only some players support this container.'
-                            % info_dict['id'])
+                            % ('m4a' if info_dict.get('container') == 'm4a_dash' else 'mp4',
+                                filename))
                     elif fixup_policy == 'detect_or_warn':
-                        fixup_pp = FFmpegFixupM4aPP(self)
+                        fixup_pp = FFmpegFixupMpegDashPP(self)
                         if fixup_pp.available:
                             info_dict.setdefault('__postprocessors', [])
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: writing DASH m4a. '
+                                'Writing DASH %s in "%s". '
                                 'Only some players support this container. %s'
-                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
+                                % ('m4a' if info_dict.get('container') == 'm4a_dash' else 'mp4',
+                                    filename, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 3ea5183999d..b15d04a642d 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -7,7 +7,7 @@
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
     FFmpegFixupM3u8PP,
-    FFmpegFixupM4aPP,
+    FFmpegFixupMpegDashPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -28,7 +28,7 @@ def get_postprocessor(key):
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
     'FFmpegFixupM3u8PP',
-    'FFmpegFixupM4aPP',
+    'FFmpegFixupMpegDashPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9f76c9d4ed2..a7297a11098 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -564,12 +564,18 @@ def run(self, info):
         return [], info
 
 
-class FFmpegFixupM4aPP(FFmpegPostProcessor):
+class FFmpegFixupMpegDashPP(FFmpegPostProcessor):
     def run(self, info):
-        if info.get('container') != 'm4a_dash':
+        if info.get('container') not in ('m4a_dash', 'mp4_dash'):
             return [], info
 
         filename = info['filepath']
+        if info.get('container') == 'mp4_dash':
+            with open(encodeFilename(filename), 'rb') as f:
+                b = f.read(8)
+            if b[4:8] == b'ftyp':
+                return [], info
+
         temp_filename = prepend_extension(filename, 'temp')
 
         options = ['-c', 'copy', '-f', 'mp4']

From d3725a6ab25bc04ded72a7eed7d8edcb8eef6091 Mon Sep 17 00:00:00 2001
From: Oliver Pfeiffer <olipfei@netscape.net>
Date: Wed, 1 Dec 2021 21:50:40 +0100
Subject: [PATCH 074/394] [douyin] Add new extractor

---
 youtube_dl/extractor/douyin.py     | 94 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 98 insertions(+)
 create mode 100644 youtube_dl/extractor/douyin.py

diff --git a/youtube_dl/extractor/douyin.py b/youtube_dl/extractor/douyin.py
new file mode 100644
index 00000000000..6a3324063f6
--- /dev/null
+++ b/youtube_dl/extractor/douyin.py
@@ -0,0 +1,94 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    try_get,
+    orderedSet
+)
+
+
+class DouyinVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.douyin.com/video/6989098563519270181',
+        'md5': 'b4c99f3be1be343948a9eaaea8303d27',
+        'info_dict': {
+            'id': '6989098563519270181',
+            'url': r're:^https://v3-dy-o.zjcdn.com/[0-9a-f]{32}/[0-9a-f]{8}/video/tos/cn/tos-cn-ve-15/[0-9a-f]{32}/?.*',
+            'title': '杨集#我的家乡 ',
+            'uploader': '永恒的爱',
+            'uploader_id': '104081949894',
+            'timestamp': 1627276320000.0,
+            'ext': 'mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        iteminfo = self._download_json('https://www.douyin.com/web/api/v2/aweme/iteminfo',
+                                       video_id, query={'item_ids': video_id})
+        if iteminfo['status_code']:
+            raise ExtractorError(iteminfo['status_msg'], video_id=video_id)
+
+        item_list = iteminfo.get('item_list')
+        if not item_list:
+            raise ExtractorError('The video you want to download does not exist any more',
+                                 video_id=video_id, expected=True)
+        item = item_list[0]
+
+        info_dict = {}
+        info_dict['id'] = video_id
+        info_dict['title'] = item['desc']
+        info_dict['url'] = item['video']['play_addr']['url_list'][0]
+        info_dict['uploader'] = try_get(item, lambda x: x['author']['nickname'])
+        info_dict['uploader_id'] = try_get(item, lambda x: x['author']['uid'])
+        info_dict['duration'] = float_or_none(item.get('duration') or try_get(item, lambda x: x['video']['duration'], int), scale=1000)
+        info_dict['timestamp'] = float_or_none(item.get('create_time'), invscale=1000)
+        info_dict['width'] = try_get(item, lambda x: x['video']['width'])
+        info_dict['height'] = try_get(item, lambda x: x['video']['height'])
+        info_dict['vbr'] = try_get(item, lambda x: x['video']['bit_rate'])
+        info_dict['ext'] = 'mp4'
+
+        return info_dict
+
+
+class DouyinUserIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?douyin\.com/user/(?P<id>[a-zA-Z0-9_\-]+)'
+    _TEST = {
+        'url': 'https://www.douyin.com/user/MS4wLjABAAAAP5Q7Z-SwleIzAACYIu-LrwGbEZzN2dc5PT3hGNSTkSM',
+        'info_dict': {
+            'id': 'MS4wLjABAAAAP5Q7Z-SwleIzAACYIu-LrwGbEZzN2dc5PT3hGNSTkSM'
+        },
+        'playlist_mincount': 31
+    }
+
+    def _real_extract(self, url):
+        sec_uid = self._match_id(url)
+
+        has_more = True
+        max_cursor = ''
+
+        entries = []
+        while has_more:
+            post = self._download_json('https://www.douyin.com/web/api/v2/aweme/post',
+                                       sec_uid, query={'sec_uid': sec_uid, 'max_cursor': max_cursor, 'count': 50})
+            if post['status_code']:
+                raise ExtractorError(post['status_msg'], video_id=sec_uid)
+
+            aweme_list = post.get('aweme_list')
+            if aweme_list is None:
+                raise ExtractorError('JSON response does not contain aweme_list', video_id=sec_uid)
+
+            entries.extend([self.url_result('https://www.douyin.com/video/%s' % aweme['aweme_id'],
+                                            ie=DouyinVideoIE.ie_key(), video_id=aweme['aweme_id'])
+                            for aweme in aweme_list])
+
+            has_more = post['has_more']
+            max_cursor = post['max_cursor']
+
+        return self.playlist_result(orderedSet(entries), sec_uid)
+
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..a77a1ccb697 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -290,6 +290,10 @@
 from .dhm import DHMIE
 from .digg import DiggIE
 from .dotsub import DotsubIE
+from .douyin import (
+    DouyinVideoIE,
+    DouyinUserIE
+)
 from .douyutv import (
     DouyuShowIE,
     DouyuTVIE,

From 3b9b1cd4314f9d44630a2a65abfff8abab454887 Mon Sep 17 00:00:00 2001
From: lanegramling <lanegramling@gmail.com>
Date: Tue, 14 Dec 2021 21:04:03 -0700
Subject: [PATCH 075/394] [youtube] Fix function signature parser (refs
 ytdl-org/#30363)

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e61157bfafc..de73fecc857 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1335,10 +1335,10 @@ def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
-             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
+             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',

From 32f049b37a08cb092190f63a0276155ca54a5a44 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 076/394] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index de73fecc857..6f3162decd6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From afaec67d45df19642eb4af230b506127af84ceff Mon Sep 17 00:00:00 2001
From: scramblr <scramblr@blackhats.ru>
Date: Thu, 16 Dec 2021 18:04:23 -0800
Subject: [PATCH 077/394] Fixes 404 Issue Currently Present On XVIDEOS.COM

This simple addition of '0' fixes the 404 issues many are having with downloading videos from xvideos.com. Tested on linux, windows, and macOS.
---
 youtube_dl/extractor/xvideos.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 8fc64914caf..e63d4690d4b 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -82,7 +82,7 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.xvideos.com/video%s/' % video_id, video_id)
+            'https://www.xvideos.com/video%s/0' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:

From 3b24b4a599408d62fc20240cdc6fac5b9d612071 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:43:16 +0700
Subject: [PATCH 078/394] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 680fffdf8bc..e530e6aead2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
+
+Extractors
+* [youtube] Update signature function patterns (#30363, #30366)
+* [peertube] Only call description endpoint if necessary (#29383)
+* [periscope] Pass referer to HLS requests (#29419)
+- [liveleak] Remove extractor (#17625, #24222, #29331)
++ [pornhub] Add support for pornhubthbh7ap3u.onion
+* [pornhub] Detect geo restriction
+* [pornhub] Dismiss tbr extracted from download URLs (#28927)
+* [curiositystream:collection] Extend _VALID_URL (#26326, #29117)
+* [youtube] Make get_video_info processing more robust (#29333)
+* [youtube] Workaround for get_video_info request (#29333)
+* [bilibili] Strip uploader name (#29202)
+* [youtube] Update invidious instance list (#29281)
+* [umg:de] Update GraphQL API URL (#29304)
+* [nrk] Switch psapi URL to https (#29344)
++ [egghead] Add support for app.egghead.io (#28404, #29303)
+* [appleconnect] Fix extraction (#29208)
++ [orf:tvthek] Add support for MPD formats (#28672, #29236)
+
+
 version 2021.06.06
 
 Extractors

From 830fb46973fe55c2f7b5ed315d56609f206c10cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:49:07 +0700
Subject: [PATCH 079/394] release 2021.12.17

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 2 --
 youtube_dl/version.py                            | 2 +-
 8 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 4eb50523189..e5405c23590 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 9fed0b489e8..33b01ce7fda 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 573e8ded0ab..285610cc70e 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index c0031bf7a32..af73525fbc9 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 1138ab2ca7e..42c878b83af 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index e530e6aead2..6588642825b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.12.17
 
 Core
 * [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ed0d5e9d990..ae2a6b8b02b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -472,8 +472,6 @@
  - **LinuxAcademy**
  - **LiTV**
  - **LiveJournal**
- - **LiveLeak**
- - **LiveLeakEmbed**
  - **livestream**
  - **livestream:original**
  - **LnkGo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 461dd87cafe..b82fbc702ff 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.06.06'
+__version__ = '2021.12.17'

From d577b929aa83b9a0717f5c19c45dfaf1e12454d8 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Tue, 2 Nov 2021 11:18:39 +0000
Subject: [PATCH 080/394] Back-port JS interpreter upgrade from yt-dlp PR #1437

---
 test/test_jsinterp.py  |  51 +++++
 youtube_dl/compat.py   |   5 +
 youtube_dl/jsinterp.py | 504 ++++++++++++++++++++++++++++++++---------
 3 files changed, 453 insertions(+), 107 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c24b8ca742a..4d05ea61013 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -112,6 +112,57 @@ def test_call(self):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        # function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 29e0d3a0292..2004a405a8e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,10 @@
 import sys
 import xml.etree.ElementTree
 
+try:
+    import collections.abc as compat_collections_abc
+except ImportError:
+    import collections as compat_collections_abc
 
 try:
     import urllib.request as compat_urllib_request
@@ -3025,6 +3029,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_b64decode',
     'compat_basestring',
     'compat_chr',
+    'compat_collections_abc',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7bda596102a..061e92c2a4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -8,6 +8,15 @@
     ExtractorError,
     remove_quotes,
 )
+from .compat import (
+    compat_collections_abc
+)
+MutableMapping = compat_collections_abc.MutableMapping
+
+
+class Nonlocal:
+    pass
+
 
 _OPERATORS = [
     ('|', operator.or_),
@@ -22,11 +31,55 @@
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
 
+class JS_Break(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid break')
+
+
+class JS_Continue(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid continue')
+
+
+class LocalNameSpace(MutableMapping):
+    def __init__(self, *stack):
+        self.stack = tuple(stack)
+
+    def __getitem__(self, key):
+        for scope in self.stack:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        for scope in self.stack:
+            if key in scope:
+                scope[key] = value
+                break
+        else:
+            self.stack[0][key] = value
+        return value
+
+    def __delitem__(self, key):
+        raise NotImplementedError('Deleting is not supported')
+
+    def __iter__(self):
+        for scope in self.stack:
+            for scope_item in iter(scope):
+                yield scope_item
+
+    def __len__(self, key):
+        return len(iter(self))
+
+    def __repr__(self):
+        return 'LocalNameSpace%s' % (self.stack, )
+
+
 class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
@@ -34,11 +87,58 @@ def __init__(self, code, objects=None):
         self.code = code
         self._functions = {}
         self._objects = objects
+        self.__named_object_counter = 0
+
+    def _named_object(self, namespace, obj):
+        self.__named_object_counter += 1
+        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        namespace[name] = obj
+        return name
+
+    @staticmethod
+    def _separate(expr, delim=',', max_split=None):
+        if not expr:
+            return
+        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
+        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        for idx, char in enumerate(expr):
+            if char in parens:
+                parens[char] += 1
+            is_in_parens = (parens['['] - parens[']']
+                            or parens['('] - parens[')']
+                            or parens['{'] - parens['}'])
+            if char == delim[pos] and not is_in_parens:
+                if pos == max_pos:
+                    pos = 0
+                    yield expr[start: idx - max_pos]
+                    start = idx + 1
+                    splits += 1
+                    if max_split and splits >= max_split:
+                        break
+                else:
+                    pos += 1
+            else:
+                pos = 0
+        yield expr[start:]
+
+    @staticmethod
+    def _separate_at_paren(expr, delim):
+        separated = list(JSInterpreter._separate(expr, delim, 1))
+        if len(separated) < 2:
+            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+        return separated[0][1:].strip(), separated[1].strip()
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
+        sub_statements = list(self._separate(stmt, ';'))
+        stmt = (sub_statements or ['']).pop()
+        for sub_stmt in sub_statements:
+            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+
         should_abort = False
         stmt = stmt.lstrip()
         stmt_m = re.match(r'var\s', stmt)
@@ -61,25 +161,119 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr == '':  # Empty expression
             return None
 
+        if expr.startswith('{'):
+            inner, outer = self._separate_at_paren(expr, '}')
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            if not outer or should_abort:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
         if expr.startswith('('):
-            parens_count = 0
-            for m in re.finditer(r'[()]', expr):
-                if m.group(0) == '(':
-                    parens_count += 1
+            inner, outer = self._separate_at_paren(expr, ')')
+            inner = self.interpret_expression(inner, local_vars, allow_recursion)
+            if not outer:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('['):
+            inner, outer = self._separate_at_paren(expr, ']')
+            name = self._named_object(local_vars, [
+                self.interpret_expression(item, local_vars, allow_recursion)
+                for item in self._separate(inner)])
+            expr = name + outer
+
+        m = re.match(r'try\s*', expr)
+        if m:
+            if expr[m.end()] == '{':
+                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            else:
+                try_expr, expr = expr[m.end() - 1:], ''
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('catch'):
+            # We ignore the catch block
+            _, expr = self._separate_at_paren(expr, '}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('for'):
+            def raise_constructor_error(c):
+                raise ExtractorError(
+                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
+
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            if remaining.startswith('{'):
+                body, expr = self._separate_at_paren(remaining, '}')
+            else:
+                m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if m:
+                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                    body, expr = self._separate_at_paren(remaining, '}')
+                    body = 'switch(%s){%s}' % (switch_val, body)
                 else:
-                    parens_count -= 1
-                    if parens_count == 0:
-                        sub_expr = expr[1:m.start()]
-                        sub_result = self.interpret_expression(
-                            sub_expr, local_vars, allow_recursion)
-                        remaining_expr = expr[m.end():].strip()
-                        if not remaining_expr:
-                            return sub_result
-                        else:
-                            expr = json.dumps(sub_result) + remaining_expr
+                    body, expr = remaining, ''
+            start, cndn, increment = self._separate(constructor, ';')
+            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
+                raise_constructor_error(constructor)
+            while True:
+                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                    break
+                try:
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    if should_abort:
+                        return ret
+                except JS_Break:
+                    break
+                except JS_Continue:
+                    pass
+                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
+                    raise_constructor_error(constructor)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('switch'):
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
+            body, expr = self._separate_at_paren(remaining, '}')
+            body, default = body.split('default:') if 'default:' in body else (body, None)
+            items = body.split('case ')[1:]
+            if default:
+                items.append('default:%s' % (default, ))
+            matched = False
+            for item in items:
+                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
+                if matched:
+                    try:
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        if should_abort:
+                            return ret
+                    except JS_Break:
                         break
-            else:
-                raise ExtractorError('Premature end of parens in %r' % expr)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        # Comma separated statements
+        sub_expressions = list(self._separate(expr))
+        expr = sub_expressions.pop().strip() if sub_expressions else ''
+        for sub_expr in sub_expressions:
+            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+
+        for m in re.finditer(r'''(?x)
+                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
+                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+            var = m.group('var1') or m.group('var2')
+            start, end = m.span()
+            sign = m.group('pre_sign') or m.group('post_sign')
+            ret = local_vars[var]
+            local_vars[var] += 1 if sign[0] == '+' else -1
+            if m.group('pre_sign'):
+                ret = local_vars[var]
+            expr = expr[:start] + json.dumps(ret) + expr[end:]
 
         for op, opfunc in _ASSIGN_OPERATORS:
             m = re.match(r'''(?x)
@@ -88,14 +282,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
                 (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
             if not m:
                 continue
-            right_val = self.interpret_expression(
-                m.group('expr'), local_vars, allow_recursion - 1)
+            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
 
             if m.groupdict().get('index'):
                 lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(
-                    m.group('index'), local_vars, allow_recursion)
-                assert isinstance(idx, int)
+                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+                if not isinstance(idx, int):
+                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -109,8 +302,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr.isdigit():
             return int(expr)
 
+        if expr == 'break':
+            raise JS_Break()
+        elif expr == 'continue':
+            raise JS_Continue()
+
         var_m = re.match(
-            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
             expr)
         if var_m:
             return local_vars[var_m.group('name')]
@@ -124,91 +322,161 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
             r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
+            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
             return val[idx]
 
+        def raise_expr_error(where, op, exp):
+            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+
+        for op, opfunc in _OPERATORS:
+            separated = list(self._separate(expr, op))
+            if len(separated) < 2:
+                continue
+            right_val = separated.pop()
+            left_val = op.join(separated)
+            left_val, should_abort = self.interpret_statement(
+                left_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('left-side', op, expr)
+            right_val, should_abort = self.interpret_statement(
+                right_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('right-side', op, expr)
+            return opfunc(left_val or 0, right_val)
+
         m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = remove_quotes(m.group('member') or m.group('member2'))
-            arg_str = m.group('args')
+            nl = Nonlocal()
 
-            if variable in local_vars:
-                obj = local_vars[variable]
-            else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
-
-            if arg_str is None:
-                # Member access
-                if member == 'length':
-                    return len(obj)
-                return obj[member]
-
-            assert expr.endswith(')')
-            # Function call
-            if arg_str == '':
-                argvals = tuple()
+            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+            arg_str = expr[m.end():]
+            if arg_str.startswith('('):
+                arg_str, remaining = self._separate_at_paren(arg_str, ')')
             else:
-                argvals = tuple([
+                arg_str, remaining = None, arg_str
+
+            def assertion(cndn, msg):
+                """ assert, but without risk of getting optimized out """
+                if not cndn:
+                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+
+            def eval_method():
+                # nonlocal member
+                member = nl.member
+                if variable == 'String':
+                    obj = str
+                elif variable in local_vars:
+                    obj = local_vars[variable]
+                else:
+                    if variable not in self._objects:
+                        self._objects[variable] = self.extract_object(variable)
+                    obj = self._objects[variable]
+
+                if arg_str is None:
+                    # Member access
+                    if member == 'length':
+                        return len(obj)
+                    return obj[member]
+
+                # Function call
+                argvals = [
                     self.interpret_expression(v, local_vars, allow_recursion)
-                    for v in arg_str.split(',')])
-
-            if member == 'split':
-                assert argvals == ('',)
-                return list(obj)
-            if member == 'join':
-                assert len(argvals) == 1
-                return argvals[0].join(obj)
-            if member == 'reverse':
-                assert len(argvals) == 0
-                obj.reverse()
-                return obj
-            if member == 'slice':
-                assert len(argvals) == 1
-                return obj[argvals[0]:]
-            if member == 'splice':
-                assert isinstance(obj, list)
-                index, howMany = argvals
-                res = []
-                for i in range(index, min(index + howMany, len(obj))):
-                    res.append(obj.pop(index))
-                return res
-
-            return obj[member](argvals)
-
-        for op, opfunc in _OPERATORS:
-            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
-            if not m:
-                continue
-            x, abort = self.interpret_statement(
-                m.group('x'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature left-side return of %s in %r' % (op, expr))
-            y, abort = self.interpret_statement(
-                m.group('y'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature right-side return of %s in %r' % (op, expr))
-            return opfunc(x, y)
+                    for v in self._separate(arg_str)]
+
+                if obj == str:
+                    if member == 'fromCharCode':
+                        assertion(argvals, 'takes one or more arguments')
+                        return ''.join(map(chr, argvals))
+                    raise ExtractorError('Unsupported string method %s' % (member, ))
+
+                if member == 'split':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(argvals == [''], 'with arguments is not implemented')
+                    return list(obj)
+                elif member == 'join':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return argvals[0].join(obj)
+                elif member == 'reverse':
+                    assertion(not argvals, 'does not take any arguments')
+                    obj.reverse()
+                    return obj
+                elif member == 'slice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return obj[argvals[0]:]
+                elif member == 'splice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    index, howMany = (argvals + [len(obj)])[:2]
+                    if index < 0:
+                        index += len(obj)
+                    add_items = argvals[2:]
+                    res = []
+                    for i in range(index, min(index + howMany, len(obj))):
+                        res.append(obj.pop(index))
+                    for i, item in enumerate(add_items):
+                        obj.insert(index + i, item)
+                    return res
+                elif member == 'unshift':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    for item in reversed(argvals):
+                        obj.insert(0, item)
+                    return obj
+                elif member == 'pop':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(not argvals, 'does not take any arguments')
+                    if not obj:
+                        return
+                    return obj.pop()
+                elif member == 'push':
+                    assertion(argvals, 'takes one or more arguments')
+                    obj.extend(argvals)
+                    return obj
+                elif member == 'forEach':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    f, this = (argvals + [''])[:2]
+                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                elif member == 'indexOf':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    idx, start = (argvals + [0])[:2]
+                    try:
+                        return obj.index(idx, start)
+                    except ValueError:
+                        return -1
+
+                if isinstance(obj, list):
+                    member = int(member)
+                    nl.member = member
+                return obj[member](argvals)
+
+            if remaining:
+                return self.interpret_expression(
+                    self._named_object(local_vars, eval_method()) + remaining,
+                    local_vars, allow_recursion)
+            else:
+                return eval_method()
 
-        m = re.match(
-            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
+        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
-                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
-            if fname not in self._functions:
+                for v in self._separate(m.group('args'))])
+            if fname in local_vars:
+                return local_vars[fname](argvals)
+            elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
-        raise ExtractorError('Unsupported JS expression %r' % expr)
+        if expr:
+            raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -233,30 +501,52 @@ def extract_object(self, objname):
 
         return obj
 
-    def extract_function(self, funcname):
+    def extract_function_code(self, funcname):
+        """ @returns argnames, code """
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
-                \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
+                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
             self.code)
+        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
-        argnames = func_m.group('args').split(',')
+        return func_m.group('args').split(','), code
 
-        return self.build_function(argnames, func_m.group('code'))
+    def extract_function(self, funcname):
+        return self.extract_function_from_code(*self.extract_function_code(funcname))
+
+    def extract_function_from_code(self, argnames, code, *global_stack):
+        local_vars = {}
+        while True:
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            if mobj is None:
+                break
+            start, body_start = mobj.span()
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(
+                local_vars,
+                self.extract_function_from_code(
+                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    body, local_vars, *global_stack))
+            code = code[:start] + name + remaining
+        return self.build_function(argnames, code, local_vars, *global_stack)
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
-        return f(args)
-
-    def build_function(self, argnames, code):
-        def resf(args):
-            local_vars = dict(zip(argnames, args))
-            for stmt in code.split(';'):
-                res, abort = self.interpret_statement(stmt, local_vars)
-                if abort:
+        return self.extract_function(funcname)(args)
+
+    def build_function(self, argnames, code, *global_stack):
+        global_stack = list(global_stack) or [{}]
+        local_vars = global_stack.pop(0)
+
+        def resf(args, **kwargs):
+            local_vars.update(dict(zip(argnames, args)))
+            local_vars.update(kwargs)
+            var_stack = LocalNameSpace(local_vars, *global_stack)
+            for stmt in self._separate(code.replace('\n', ''), ';'):
+                ret, should_abort = self.interpret_statement(stmt, var_stack)
+                if should_abort:
                     break
-            return res
+            return ret
         return resf

From 116500e2ba6c504008985163bed20d8f4c7c88e7 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 4 Nov 2021 12:48:06 +0000
Subject: [PATCH 081/394] Handle default in switch better

Add https://github.com/yt-dlp/yt-dlp/commit/a1fc7ca0743c8df06416e68ee74b64e07dfe7135
Thanks coletdjnz
---
 test/test_jsinterp.py  | 15 +++++++++++++++
 youtube_dl/jsinterp.py | 23 ++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d05ea61013..acdabffb1f8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -133,6 +133,21 @@ def test_switch(self):
         self.assertEqual(jsi.call_function('x', 3), 6)
         self.assertEqual(jsi.call_function('x', 5), 0)
 
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
     def test_try(self):
         jsi = JSInterpreter('''
         function x() { try{return 10} catch(e){return 5} }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 061e92c2a4a..c35765702a6 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,21 +240,26 @@ def raise_constructor_error(c):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
-            body, default = body.split('default:') if 'default:' in body else (body, None)
-            items = body.split('case ')[1:]
-            if default:
-                items.append('default:%s' % (default, ))
-            matched = False
-            for item in items:
-                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
-                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
-                if matched:
+            items = body.replace('default:', 'case default:').split('case ')[1:]
+            for default in (False, True):
+                matched = False
+                for item in items:
+                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    if default:
+                        matched = matched or case == 'default'
+                    elif not matched:
+                        matched = (case != 'default'
+                                   and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
+                    if not matched:
+                        continue
                     try:
                         ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
+                if matched:
+                    break
             return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
 
         # Comma separated statements

From e74c63be530a27d692b65ec233c9c22d97fd7a74 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 02:06:13 +0000
Subject: [PATCH 082/394] Fix splice to handle float

Needed for new youtube js player f1ca6900
Add https://github.com/yt-dlp/yt-dlp/commit/57dbe8077f8d00e0fffac53669f40cd7d584474f#diff-729b57caa8d006426f6a8960c061f519a8b6658682284015e069745af52ffb07
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c35765702a6..c75cf45b954 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -416,7 +416,7 @@ def eval_method():
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')
-                    index, howMany = (argvals + [len(obj)])[:2]
+                    index, howMany = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
                     add_items = argvals[2:]

From ff2dcd5b428016058706d2695c9b0afc8edfd14d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 03:18:29 +0000
Subject: [PATCH 083/394] Back-port test_youtube_signature.py from yt-dlp and
 fix JSInterp accordingly

---
 test/test_youtube_signature.py | 87 ++++++++++++++++++++++++----------
 youtube_dl/jsinterp.py         |  9 ++--
 2 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 627d4cb9255..c8e85b5005a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -14,9 +14,10 @@
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.jsinterp import JSInterpreter
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
@@ -95,35 +115,54 @@ def setUp(self):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
-    assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+def t_factory(name, sig_func, url_pattern):
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
+
+        def test_func(self):
+            basename = 'player-{0}-{1}.js'.format(name, test_id)
+            fn = os.path.join(self.TESTDATA_DIR, basename)
+
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
+
+        test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
+
 
-    def test_func(self):
-        basename = 'player-%s.js' % test_id
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
 
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
-        with io.open(fn, encoding='utf-8') as testf:
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
+def n_sig(jscode, sig_input):
+    # Pending implementation of _extract_n_function_name() or similar in
+    # youtube.py, hard-code here
+    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    import re
+    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
+    funcname = funcname and funcname.group(1)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
 
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
 
-for test_spec in _TESTS:
-    make_tfunc(*test_spec)
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c75cf45b954..a2306557b4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -9,7 +9,8 @@
     remove_quotes,
 )
 from .compat import (
-    compat_collections_abc
+    compat_collections_abc,
+    compat_str,
 )
 MutableMapping = compat_collections_abc.MutableMapping
 
@@ -372,7 +373,7 @@ def eval_method():
                 # nonlocal member
                 member = nl.member
                 if variable == 'String':
-                    obj = str
+                    obj = compat_str
                 elif variable in local_vars:
                     obj = local_vars[variable]
                 else:
@@ -391,7 +392,7 @@ def eval_method():
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == str:
+                if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
@@ -533,7 +534,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    [x.strip() for x in mobj.group('args').split(',')],
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)

From c3b0045720dbd47edfc0c30eec15a6f068836a2f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Dec 2021 19:14:54 +0000
Subject: [PATCH 084/394] Refactor JSInterpreter._separate

yt-dlp/yt-dlp/@06dfe0a, improve _MATCHING_PARENS
---
 youtube_dl/jsinterp.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a2306557b4a..8eaa911cdab 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -36,6 +36,8 @@ class Nonlocal:
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
+_MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+
 
 class JS_Break(ExtractorError):
     def __init__(self):
@@ -100,26 +102,24 @@ def _named_object(self, namespace, obj):
     def _separate(expr, delim=',', max_split=None):
         if not expr:
             return
-        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
-        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         for idx, char in enumerate(expr):
-            if char in parens:
-                parens[char] += 1
-            is_in_parens = (parens['['] - parens[']']
-                            or parens['('] - parens[')']
-                            or parens['{'] - parens['}'])
-            if char == delim[pos] and not is_in_parens:
-                if pos == max_pos:
-                    pos = 0
-                    yield expr[start: idx - max_pos]
-                    start = idx + 1
-                    splits += 1
-                    if max_split and splits >= max_split:
-                        break
-                else:
-                    pos += 1
-            else:
+            if char in _MATCHING_PARENS:
+                counters[_MATCHING_PARENS[char]] += 1
+            elif char in counters:
+                counters[char] -= 1
+            if char != delim[pos] or any(counters.values()):
                 pos = 0
+                continue
+            elif pos != delim_len:
+                pos += 1
+                continue
+            yield expr[start: idx - delim_len]
+            start, pos = idx + 1, 0
+            splits += 1
+            if max_split and splits >= max_split:
+                break
         yield expr[start:]
 
     @staticmethod

From 905d1d281ddfa5d183fc445010d350cefc6a58ec Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 17 Dec 2021 14:36:42 +0000
Subject: [PATCH 085/394] Implement n-param descrambling using JSInterp

---
 youtube_dl/extractor/youtube.py | 326 +++++---------------------------
 1 file changed, 46 insertions(+), 280 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6f3162decd6..63918924df8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1385,292 +1385,62 @@ def _extract_player_url(self, webpage):
                 'https://www.youtube.com', player_url)
         return player_url
 
-    # Based on an equivalent function [2] in the youtube.lua script from VLC
-    # Many thanks to @linkfanel [3]
-    # NB This code could fail if YT should revise the player code and would then have
-    # to be reworked (thankless task previously undertaken at [1] and [2])
+    # from yt-dlp
+    # See also:
     # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
-    def _n_descramble(self, n_param, js):
+    def _extract_n_function_name(self, jscode):
+        return self._search_regex(
+            (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
+            jscode, 'Initial JS player n function name', group='nfunc')
+
+    def _extract_n_function(self, video_id, player_url):
+        player_id = self._extract_player_info(player_url)
+        func_code = self._downloader.cache.load('youtube-nsig', player_id)
+
+        if func_code:
+            jsi = JSInterpreter(func_code)
+        else:
+            player_id = self._extract_player_info(player_url)
+            jscode = self._get_player_code(video_id, player_url, player_id)
+            funcname = self._extract_n_function_name(jscode)
+            jsi = JSInterpreter(jscode)
+            func_code = jsi.extract_function_code(funcname)
+            self._downloader.cache.store('youtube-nsig', player_id, func_code)
+
+        if self._downloader.params.get('youtube_print_sig_code'):
+            self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(player_id, func_code[1]))
+
+        return lambda s: jsi.extract_function_from_code(*func_code)([s])
+
+    def _n_descramble(self, n_param, player_url, video_id):
         """Compute the response to YT's "n" parameter challenge
 
         Args:
-        n_param -- challenge string that is the value of the
-                   URL's "n" query parameter
-        js      -- text of the JS player code that includes the
-                   challenge response algorithm
+        n_param     -- challenge string that is the value of the
+                       URL's "n" query parameter
+        player_url  -- URL of YT player JS
+        video_id
         """
-        if not js:
-            return
-
-        # helper functions (part 1)
-        def isiterable(x):
-            try:
-                return x.__getitem__ and True
-            except AttributeError:
-                return False
 
-        def find_first(pattern, string, flags=0, groups=1):
-            pattern = re.compile(pattern, flags)
-            return next(map(lambda m: m.groups() if groups is True else m.group(groups),
-                            pattern.finditer(string)),
-                        (None, ) * pattern.groups if groups is True else None)
-
-        # Look for the descrambler function's name
-        # a.D&&(b=a.get("n"))&&(b=lha(b),a.set("n",b))}};
-        descrambler = find_first(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', js)
-        if not descrambler:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling function name")
-            return
-        # Fetch the code of the descrambler function
-        # lha=function(a){var b=a.split(""),c=[310282131,"KLf3",b,null,function(d,e){d.push(e)},-45817231, [data and transformations...] ,1248130556];c[3]=c;c[15]=c;c[18]=c;try{c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [scripted calls...] ,c[25](c[33],c[3])}catch(d){return"enhanced_except_4ZMBnuz-_w8_"+a}return b.join("")};
-        code = find_first(r'(?s)%s=function\([^)]+\)\{(.+?)\};' % (descrambler, ), js)
-        if not code:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling code")
-            return
-        # Split code into two main sections: 1/ data and transformations,
-        # and 2/ a script of calls
-        datac, xc, script = find_first(r'(?s)c=\[(.+)\];(.+?);try\{(.+)\}catch\(', code, groups=True)
-        if not datac or not script:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
-            return
-        if xc:
-            # ad hoc update of c[] array after declaration (player 2dfe380c, ...)
-            xc = [int(x) for x in re.findall(r'c\[(\d+)\]=c\b', xc)]
-            if not xc:
-                self.report_warning("Couldn't extract YouTube video throttling parameter additional rules")
-        else:
-            xc = []
-
-        # Split "n" parameter into a table as descrambling operates on it
-        # as one of several arrays - in Python just copy it as a list
-        n = list(n_param)
-        # Helper: table_len = function() ... end - in Python just use len
-
-        # Common routine shared by the compound transformations,
-        # compounding the "n" parameter with an input string,
-        # character by character using a Base64 alphabet.
-        # d.forEach(function(l,m,n){this.push(n[m]=h[(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length])},e.split(""))
-        def compound(ntab, strg, alphabet, charcode):
-            if ntab != n or type(strg) != compat_str:
-                return True
-            inp = list(strg)
-            llen = len(alphabet)
-            ntab_copy = ntab[:]
-            for i, c in enumerate(ntab_copy):
-                if type(c) != compat_str:
-                    return True
-                pos1 = alphabet.find(c)
-                pos2 = alphabet.find(inp[i])
-                if pos1 < 0 or pos2 < 0:
-                    return True
-                pos = (pos1 - pos2 + charcode - 32) % llen
-                newc = alphabet[pos]
-                ntab[i] = newc
-                inp.append(newc)
-
-        # The data section contains among others function code for a number
-        # of transformations, most of which are basic array operations.
-        # We can match these functions' code to identify them, and emulate
-        # the corresponding transformations.
-
-        # helper fns (in-place)
-        def swap(s, i, j):
-            x = s[i]
-            s[i] = s[j]
-            s[j] = x
-
-        def rotate(s, i):
-            tmp = s[:]
-            tmp[i:] = s
-            tmp[:i] = tmp[len(s):]
-            s[:] = tmp[:len(s)]
-
-        def remove(s, i):
-            del s[i]
-
-        b64_alphabets = [
-            "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
-            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
-        ]
+        sig_id = ('nsig_value', n_param)
+        if sig_id in self._player_cache:
+            return self._player_cache[sig_id]
 
-        # Compounding functions use a subfunction, so we need to be
-        # more specific in how much parsed data we consume.
-        cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
-        def_skip = r"(?s)^.*?\},\s*(.*)$"
-
-        trans = (
-            # fn_name, fn, fn_detect_pattern, skip_re
-            ('reverse', lambda tab, _: tab.reverse(),
-                        # noqa: E127
-                        # function(d){d.reverse()}
-                        # function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])}
-                        r"^function\(d\)",
-                        def_skip),
-            ('append', lambda tab, val: tab.append(val),
-                       # noqa: E127
-                       # function(d,e){d.push(e)}
-                       r"^function\(d,e\){d\.push\(e\)\},",
-                       def_skip),
-            ('remove', lambda tab, i: remove(tab, i % len(tab)) if type(i) == int else True,
-                       # noqa: E127
-                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)}
-                       r"^[^}]+?;d\.splice\(e,1\)\},",
-                       def_skip),
-            ('swap', lambda tab, i: swap(tab, 0, i % len(tab)) if type(i) == int else True,
-                     # noqa: E127
-                     # function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f}
-                     # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}
-                     r"^[^}]+?;(?:var\sf=d\[0\];d\[0\]=d\[e\];d\[e\]=f|d\.splice\(0,1,d\.splice\(e,1,d\[0\]\)\[0\]\))\},",
-                     def_skip),
-            ('rotate', lambda tab, shift: rotate(tab, shift % len(tab)) if type(shift) == int else True,
-                       # noqa: E127
-                       # function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())}
-                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
-                       r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
-                       def_skip),
-            ('alphabet1', lambda: b64_alphabets[0],
-                          # noqa: E127
-                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
-                          r"^[^}]+?case\s58:d=96;.+?\}return",
-                          def_skip),
-            ('alphabet2', lambda: b64_alphabets[1],
-                          # noqa: E127
-                          # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
-                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 58:d-=14;case 91:case 92:case 93:continue;case 123:d=47;case 94:case 95:case 96:continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
-                          r"^[^}]+?case\s58:d(?:=44;|-=14;[^}]+\})[^}]+\}return",
-                          def_skip),
-            ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
-                         # noqa: E127
-                         # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
-                         r"^function\(\w,\w,\w\)",
-                         cp_skip),
-
-            # Compound transformations first build a variation of a
-            # Base64 alphabet, then in a common section, compound the
-            # "n" parameter with an input string, character by character.
-            ('compound1', lambda tab, s: compound(tab, s, b64_alphabets[0], 96),
-                          # noqa: E127
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
-                          r"^[^}]+?case\s58:f=96;",
-                          cp_skip),
-            ('compound2', lambda tab, s: compound(tab, s, b64_alphabets[1], 96),
-                          # noqa: E127
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
-                          r"^[^}]+?case\s58:f(?:-=14|=44);",
-                          cp_skip),
-            # Fallback
-            ('unid', lambda _, __: self.report_warning("Couldn't apply unidentified YouTube video throttling parameter transformation, aborting descrambling") or True,
-                     # noqa: E127
-                     None,
-                     def_skip),
-        )
-        # The data section actually mixes input data, reference to the
-        # "n" parameter array, and self-reference to its own array, with
-        # transformation functions used to modify itself. We parse it
-        # as such into a table.
-        data = []
-        datac += ","
-        while datac:
-            # Transformation functions
-            if re.match(r"^function\(", datac):
-                name, el, _, skip = next(
-                    itertools.dropwhile(
-                        lambda x: x[2] is not None and not re.match(x[2], datac), trans))
-                datac = find_first(skip, datac)
-            # String input data
-            elif re.match(r'^"[^"]*",', datac):
-                el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
-            # Integer input data
-            elif re.match(r'^-?\d+(?:E\d+)?,', datac):
-                el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
-                el = int(float(el) if 'E' in el else el)
-            # Reference to "n" parameter array
-            elif re.match('^b,', datac):
-                el = n
-                datac = find_first(r"(?s)^b,\s*(.*)$", datac)
-            # Replaced by self-reference to data array after its declaration
-            elif re.match('^null,', datac):
-                el = data
-                datac = find_first(r"(?s)^null,\s*(.*)$", datac)
-            else:
-                self.report_warning("Couldn't parse unidentified YouTube video throttling parameter descrambling data"
-                                    '\nNear: "%s"' % datac[:64])
-                el = False
-                # Lua tables can't contain nil values: Python can, but still use False
-                datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
-            data.append(el)
-
-        # Additional rules
-        for idx in xc:
-            data[idx] = data
-
-        # Debugging helper to print data array elements
-        def prd(el, tab=None):
-            if not el:
-                return "???"
-            elif el == n:
-                return "n"
-            elif el == data:
-                return "data"
-            elif type(el) == compat_str:
-                return '"%s"' % (el, )
-            elif type(el) == int:
-                if isiterable(tab):
-                    return "%d -> %d" % (el, el % len(tab), )
-                return "%d" % (el, )
-            else:
-                for tr in trans:
-                    if el == tr[1]:
-                        return tr[0]
-            return repr(el)
-
-        # The script section contains a series of calls to elements of
-        # the data section array onto other elements of it: calls to
-        # transformations, with a reference to the data array itself or
-        # the "n" parameter array as first argument, and often input data
-        # as a second argument. We parse and emulate those calls to follow
-        # the descrambling script.
-        # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
-        for ifunc, itab, iarg, iextra in map(lambda m: m.groups(),
-                                             # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
-                                             re.finditer(
-                                                 r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[(\d+)\]\(\))?[^)]*?\)",
-                                                 script)):
-            tab = data[int(itab)]
-            arg = iarg and data[int(iarg)]
-            if iextra:
-                alphabet = data[int(iextra)]()
-                func = lambda t, s: data[int(ifunc)](t, s, alphabet)
-            else:
-                func = data[int(ifunc)]
-
-            # Uncomment to debug transformation chain
-            # nprev = ''.join(n)
-            # dprev = ' '.join(map(prd, data))
-            # print(''.join(('"n" parameter transformation: ', prd(func), "(", prd(tab), (", " + prd(arg, tab)) if arg else '', ") ", ifunc, "(", itab, (", " + iarg) if iarg else "", ")")))
-            if not callable(func) or not isiterable(tab) or func(tab, arg):
-                self.report_warning("Invalid data type encountered during YouTube video throttling parameter descrambling transformation chain, aborting"
-                                    "\nCouldn't descramble YouTube throttling URL parameter: data transfer will be throttled")
-                self.report_warning("Couldn't process youtube video URL, please check for updates to this script")
-                break
-            # Uncomment to debug transformation chain
-            # nnew = ''.join(n)
-            # if nprev != nnew:
-            #     print('from: ' + nprev + "\nto:   " + nnew)
-            # dnew = ' '.join(map(prd, data))
-            # if dprev != dnew:
-            #     print('from: ' + dprev + "\nto:   " + dnew)
-        return ''.join(n)
+        try:
+            player_id = ('nsig', player_url)
+            if player_id not in self._player_cache:
+                self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
+            func = self._player_cache[player_id]
+            self._player_cache[sig_id] = func(n_param)
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
+            return self._player_cache[sig_id]
+        except Exception as e:
+            raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
 
     def _unthrottle_format_urls(self, video_id, player_url, formats):
-        if not player_url:
-            return
-        player_id = self._extract_player_info(player_url)
-        code = self._get_player_code(video_id, player_url, player_id)
-        n_cache = {}
         for fmt in formats:
             parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
             qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
@@ -1678,11 +1448,7 @@ def _unthrottle_format_urls(self, video_id, player_url, formats):
             if not n_param:
                 continue
             n_param = n_param[-1]
-            n_response = n_cache.get(n_param)
-            if not n_response:
-                n_response = self._n_descramble(n_param, code)
-                if n_response:
-                    n_cache[n_param] = n_response
+            n_response = self._n_descramble(n_param, player_url, video_id)
             if n_response:
                 qs['n'] = [n_response]
                 fmt['url'] = compat_urlparse.urlunparse(

From 0ec08cef90addb812b946c12f715370d2d66ffe9 Mon Sep 17 00:00:00 2001
From: Oliver Pfeiffer <olipfei@netscape.net>
Date: Fri, 17 Dec 2021 21:54:41 +0100
Subject: [PATCH 086/394] [douyin] Incorporated dirkf's remarks

---
 youtube_dl/extractor/douyin.py | 43 +++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/douyin.py b/youtube_dl/extractor/douyin.py
index 6a3324063f6..082f7e850bc 100644
--- a/youtube_dl/extractor/douyin.py
+++ b/youtube_dl/extractor/douyin.py
@@ -4,7 +4,7 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    float_or_none,
+    int_or_none,
     try_get,
     orderedSet
 )
@@ -30,9 +30,10 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         iteminfo = self._download_json('https://www.douyin.com/web/api/v2/aweme/iteminfo',
-                                       video_id, query={'item_ids': video_id})
-        if iteminfo['status_code']:
-            raise ExtractorError(iteminfo['status_msg'], video_id=video_id)
+                                       video_id, query={'item_ids': video_id}) or {}
+        status_code = iteminfo.get('status_code', 'status_code missing')
+        if status_code:
+            raise ExtractorError('%s (%s)' % (iteminfo.get('status_msg', 'status_msg missing'), status_code), video_id=video_id)
 
         item_list = iteminfo.get('item_list')
         if not item_list:
@@ -40,20 +41,19 @@ def _real_extract(self, url):
                                  video_id=video_id, expected=True)
         item = item_list[0]
 
-        info_dict = {}
-        info_dict['id'] = video_id
-        info_dict['title'] = item['desc']
-        info_dict['url'] = item['video']['play_addr']['url_list'][0]
-        info_dict['uploader'] = try_get(item, lambda x: x['author']['nickname'])
-        info_dict['uploader_id'] = try_get(item, lambda x: x['author']['uid'])
-        info_dict['duration'] = float_or_none(item.get('duration') or try_get(item, lambda x: x['video']['duration'], int), scale=1000)
-        info_dict['timestamp'] = float_or_none(item.get('create_time'), invscale=1000)
-        info_dict['width'] = try_get(item, lambda x: x['video']['width'])
-        info_dict['height'] = try_get(item, lambda x: x['video']['height'])
-        info_dict['vbr'] = try_get(item, lambda x: x['video']['bit_rate'])
-        info_dict['ext'] = 'mp4'
-
-        return info_dict
+        return {
+            'id': video_id,
+            'title': item['desc'],
+            'url': item['video']['play_addr']['url_list'][0],
+            'uploader': try_get(item, lambda x: x['author']['nickname']),
+            'uploader_id': try_get(item, lambda x: x['author']['uid']),
+            'duration': int_or_none(item.get('duration') or try_get(item, lambda x: x['video']['duration'], int), scale=1000),
+            'timestamp': int_or_none(item.get('create_time'), invscale=1000),
+            'width': try_get(item, lambda x: x['video']['width']),
+            'height': try_get(item, lambda x: x['video']['height']),
+            'vbr': try_get(item, lambda x: x['video']['bit_rate']),
+            'ext': 'mp4'
+        }
 
 
 class DouyinUserIE(InfoExtractor):
@@ -75,9 +75,10 @@ def _real_extract(self, url):
         entries = []
         while has_more:
             post = self._download_json('https://www.douyin.com/web/api/v2/aweme/post',
-                                       sec_uid, query={'sec_uid': sec_uid, 'max_cursor': max_cursor, 'count': 50})
-            if post['status_code']:
-                raise ExtractorError(post['status_msg'], video_id=sec_uid)
+                                       sec_uid, query={'sec_uid': sec_uid, 'max_cursor': max_cursor, 'count': 50}) or {}
+            status_code = post.get('status_code', 'status_code missing')
+            if status_code:
+                raise ExtractorError('%s (%s)' % (post.get('status_msg', 'status_msg missing'), status_code), video_id=sec_uid)
 
             aweme_list = post.get('aweme_list')
             if aweme_list is None:

From 76f92b3521af222ffcf34cb2c27d141eee465d8a Mon Sep 17 00:00:00 2001
From: scramblr <scramblr@blackhats.ru>
Date: Sun, 26 Dec 2021 19:06:40 -0800
Subject: [PATCH 087/394] Updated with patch by sarnoud

Updated with patch by @sarnoud via https://github.com/ytdl-org/youtube-dl/issues/29956
---
 youtube_dl/extractor/francetv.py | 51 +++++++++++++++++---------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index e4ec2e2004c..1797c83effb 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -91,29 +91,31 @@ def _extract_video(self, video_id, catalogue=None):
         # However when provided, some extra formats may be returned so we pass
         # it if available.
         info = self._download_json(
-            'https://sivideo.webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/',
+            'https://player.webservices.francetelevisions.fr/v1/videos/%s' % (video_id),
             video_id, 'Downloading video JSON', query={
-                'idDiffusion': video_id,
-                'catalogue': catalogue or '',
+                'country_code': 'FR',
+                'device_type': 'desktop',
+                'browser': 'chrome'
             })
 
         if info.get('status') == 'NOK':
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, info['message']),
                 expected=True)
-        allowed_countries = info['videos'][0].get('geoblocage')
-        if allowed_countries:
-            georestricted = True
-            geo_info = self._download_json(
-                'http://geo.francetv.fr/ws/edgescape.json', video_id,
-                'Downloading geo restriction info')
-            country = geo_info['reponse']['geo_info']['country_code']
-            if country not in allowed_countries:
-                raise ExtractorError(
-                    'The video is not available from your location',
-                    expected=True)
-        else:
-            georestricted = False
+        # allowed_countries = info['videos'][0].get('geoblocage')
+        # if allowed_countries:
+        #     georestricted = True
+        #     geo_info = self._download_json(
+        #         'http://geo.francetv.fr/ws/edgescape.json', video_id,
+        #         'Downloading geo restriction info')
+        #     country = geo_info['reponse']['geo_info']['country_code']
+        #     if country not in allowed_countries:
+        #         raise ExtractorError(
+        #             'The video is not available from your location',
+        #             expected=True)
+        # else:
+        #     georestricted = False
+        georestricted = False
 
         def sign(manifest_url, manifest_id):
             for host in ('hdfauthftv-a.akamaihd.net', 'hdfauth.francetv.fr'):
@@ -131,12 +133,13 @@ def sign(manifest_url, manifest_id):
 
         videos = []
 
-        for video in (info.get('videos') or []):
-            if video.get('statut') != 'ONLINE':
-                continue
-            if not video.get('url'):
-                continue
-            videos.append(video)
+        videos.append(info['video'])
+        # for video in (info.get('videos') or []):
+        #     if video.get('statut') != 'ONLINE':
+        #         continue
+        #     if not video.get('url'):
+        #         continue
+        #     videos.append(video)
 
         if not videos:
             for device_type in ['desktop', 'mobile']:
@@ -193,8 +196,8 @@ def sign(manifest_url, manifest_id):
 
         self._sort_formats(formats)
 
-        title = info['titre']
-        subtitle = info.get('sous_titre')
+        title = info['meta']['title']
+        subtitle = info['meta']['additional_title']
         if subtitle:
             title += ' - %s' % subtitle
         title = title.strip()

From b4eb012d3455bc9ff65df10926d384b09efd742d Mon Sep 17 00:00:00 2001
From: wranai <dakhota85@gmail.com>
Date: Wed, 10 Nov 2021 23:43:55 +0100
Subject: [PATCH 088/394] fix tiktok when logged in

tiktok now shows metadata in a diff format when logged in
---
 youtube_dl/extractor/tiktok.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index 4faa6de543f..f698e4cf4bf 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -107,10 +107,18 @@ def _real_initialize(self):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        page_props = self._parse_json(self._search_regex(
-            r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
-            webpage, 'data'), video_id)['props']['pageProps']
-        data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
+        try:
+            page_props = self._parse_json(self._search_regex(
+                r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
+                webpage, 'data'), video_id)['props']['pageProps']
+            data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
+        except:
+            page_props = self._parse_json(self._search_regex(
+                r'<script[^>]+\bid=["\']sigi-persisted-data[^>]+>window\[\'SIGI_STATE\']=({.+?});window\[',
+                webpage, 'data'), video_id)
+            data = try_get(page_props, lambda x: x['ItemModule'][video_id], dict)
+            author = try_get(page_props, lambda x: x['UserModule']['users'][data['author']], dict)
+            data['author'] = author
         if not data and page_props.get('statusCode') == 10216:
             raise ExtractorError('This video is private', expected=True)
         return self._extract_video(data, video_id)

From 37f01573a68a3b3d9a14e3b6ddf67f7abf9f6a0b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jan 2022 01:41:11 +0000
Subject: [PATCH 089/394] Add further improvements and metadata extraction for
 SIGI-type pages

See https://github.com/ytdl-org/youtube-dl/issues/30251#issuecomment-976947005
---
 youtube_dl/extractor/tiktok.py | 68 ++++++++++++++++++----------------
 1 file changed, 37 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index f698e4cf4bf..a4d9ca420fb 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -15,10 +15,11 @@
 
 class TikTokBaseIE(InfoExtractor):
     def _extract_video(self, data, video_id=None):
-        video = data['video']
-        description = str_or_none(try_get(data, lambda x: x['desc']))
-        width = int_or_none(try_get(data, lambda x: video['width']))
-        height = int_or_none(try_get(data, lambda x: video['height']))
+        video = try_get(data, lambda x: x['video'], dict)
+        if not video:
+            return
+        width = int_or_none(video.get('width'))
+        height = int_or_none(video.get('height'))
 
         format_urls = set()
         formats = []
@@ -43,30 +44,32 @@ def _extract_video(self, data, video_id=None):
         thumbnail = url_or_none(video.get('cover'))
         duration = float_or_none(video.get('duration'))
 
-        uploader = try_get(data, lambda x: x['author']['nickname'], compat_str)
-        uploader_id = try_get(data, lambda x: x['author']['id'], compat_str)
+        author = data.get('author')
+        if isinstance(author, dict):
+            uploader_id = author.get('id')
+        else:
+            uploader_id = data.get('authorId')
+            author = data
+        uploader = str_or_none(author.get('nickname'))
 
         timestamp = int_or_none(data.get('createTime'))
 
-        def stats(key):
-            return int_or_none(try_get(
-                data, lambda x: x['stats']['%sCount' % key]))
-
-        view_count = stats('play')
-        like_count = stats('digg')
-        comment_count = stats('comment')
-        repost_count = stats('share')
+        stats = try_get(data, lambda x: x['stats'], dict)
+        view_count, like_count, comment_count, repost_count = [
+            stats and int_or_none(stats.get('%sCount' % key))
+            for key in ('play', 'digg', 'comment', 'share', )]
 
         aweme_id = data.get('id') or video_id
 
         return {
             'id': aweme_id,
+            'display_id': video_id,
             'title': uploader or aweme_id,
-            'description': description,
+            'description': str_or_none(data.get('desc')),
             'thumbnail': thumbnail,
             'duration': duration,
             'uploader': uploader,
-            'uploader_id': uploader_id,
+            'uploader_id': str_or_none(uploader_id),
             'timestamp': timestamp,
             'view_count': view_count,
             'like_count': like_count,
@@ -84,11 +87,11 @@ class TikTokIE(TikTokBaseIE):
         'info_dict': {
             'id': '6606727368545406213',
             'ext': 'mp4',
-            'title': 'Zureeal',
+            'title': 'md5:24acc456b62b938a7e2dd88e978b20d9',
             'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
             'thumbnail': r're:^https?://.*',
             'duration': 15,
-            'uploader': 'Zureeal',
+            'uploader': 'md5:24acc456b62b938a7e2dd88e978b20d9',
             'uploader_id': '188294915489964032',
             'timestamp': 1538248586,
             'upload_date': '20180929',
@@ -98,7 +101,6 @@ class TikTokIE(TikTokBaseIE):
             'repost_count': int,
         }
     }]
-
     def _real_initialize(self):
         # Setup session (will set necessary cookies)
         self._request_webpage(
@@ -106,19 +108,23 @@ def _real_initialize(self):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        try:
-            page_props = self._parse_json(self._search_regex(
-                r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
-                webpage, 'data'), video_id)['props']['pageProps']
-            data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
-        except:
-            page_props = self._parse_json(self._search_regex(
-                r'<script[^>]+\bid=["\']sigi-persisted-data[^>]+>window\[\'SIGI_STATE\']=({.+?});window\[',
-                webpage, 'data'), video_id)
-            data = try_get(page_props, lambda x: x['ItemModule'][video_id], dict)
-            author = try_get(page_props, lambda x: x['UserModule']['users'][data['author']], dict)
-            data['author'] = author
+
+        page_props = self._parse_json(self._search_regex(
+            r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
+            webpage, 'sigi data', default='{}', group='json'), video_id)
+        data = try_get(page_props, lambda x: x['ItemModule'][video_id]['video'], dict)
+        if data:
+            data = page_props['ItemModule'][video_id]
+            if data.get('privateItem'):
+                raise ExtractorError('This video is private', expected=True)
+            return self._extract_video(data, video_id)
+
+        page_props = self._parse_json(self._search_regex(
+            r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
+            webpage, 'data'), video_id)['props']['pageProps']
+        data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
         if not data and page_props.get('statusCode') == 10216:
             raise ExtractorError('This video is private', expected=True)
         return self._extract_video(data, video_id)

From 09c09805688078e4e4a0d4bd3601580602d0a200 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jan 2022 01:43:52 +0000
Subject: [PATCH 090/394] Fix timeout on attempt to set up session cookies

---
 youtube_dl/extractor/tiktok.py | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index a4d9ca420fb..30130b0aceb 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -2,10 +2,14 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+    compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     float_or_none,
+    HEADRequest,
     int_or_none,
     str_or_none,
     try_get,
@@ -14,6 +18,24 @@
 
 
 class TikTokBaseIE(InfoExtractor):
+    def _download_webpage(
+            self, url_or_request, video_id, note=None, errnote=None,
+            fatal=True, tries=1, timeout=5, encoding=None, data=None,
+            headers={}, query={}, expected_status=None, setup=True):
+
+        if setup:
+            url = url_or_request.geturl() if isinstance(url_or_request, compat_urllib_request.Request) else url_or_request
+            # dummy request to set cookies
+            self._request_webpage(
+                HEADRequest(url), video_id,
+                note=False, errnote='Could not send HEAD request to %s' % url,
+                fatal=False, headers=headers)
+
+        return super(TikTokBaseIE, self)._download_webpage(
+            url_or_request, video_id, note=note, errnote=errnote,
+            fatal=fatal, tries=tries, timeout=timeout, encoding=encoding, data=data,
+            headers=headers, query=query, expected_status=expected_status)
+
     def _extract_video(self, data, video_id=None):
         video = try_get(data, lambda x: x['video'], dict)
         if not video:
@@ -101,14 +123,15 @@ class TikTokIE(TikTokBaseIE):
             'repost_count': int,
         }
     }]
-    def _real_initialize(self):
-        # Setup session (will set necessary cookies)
-        self._request_webpage(
-            'https://www.tiktok.com/', None, note='Setting up session')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        # dummy request to set cookies
+        self._request_webpage(
+            HEADRequest(url), video_id,
+            note=False, errnote='Could not send HEAD request to %s' % url,
+            fatal=False)
         webpage = self._download_webpage(url, video_id)
 
         page_props = self._parse_json(self._search_regex(

From b428754fa92ee50a2cef2ddbab019958026f88db Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jan 2022 03:18:30 +0000
Subject: [PATCH 091/394] Resurrect TikTokUserIE

---
 youtube_dl/extractor/tiktok.py | 37 +++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index 30130b0aceb..99a0c07d7e0 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -127,11 +127,6 @@ class TikTokIE(TikTokBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # dummy request to set cookies
-        self._request_webpage(
-            HEADRequest(url), video_id,
-            note=False, errnote='Could not send HEAD request to %s' % url,
-            fatal=False)
         webpage = self._download_webpage(url, video_id)
 
         page_props = self._parse_json(self._search_regex(
@@ -162,7 +157,6 @@ class TikTokUserIE(TikTokBaseIE):
         },
         'playlist_mincount': 24,
     }]
-    _WORKING = False
 
     @classmethod
     def suitable(cls, url):
@@ -170,10 +164,38 @@ def suitable(cls, url):
 
     def _real_extract(self, url):
         user_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, user_id)
+        page_props = self._parse_json(self._search_regex(
+            r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
+            webpage, 'sigi data', default='{}', group='json'), user_id)
+        user_data = try_get(page_props, lambda x: x['UserModule']['users'], dict)
+        entries = []
+        if user_data:
+            num_id = try_get(
+                user_data.values(),
+                lambda x: [user['id'] for user in x if user['uniqueId'] == user_id][0],
+                compat_str)
+            item_data = try_get(page_props, lambda x: x['ItemModule'], dict)
+            if item_data:
+                item_data = item_data.values()
+            for data in item_data or []:
+                if data.get('privateItem'):
+                    continue
+                item = self._extract_video(data, user_id)
+                if item:
+                    entries.append(item)
+            result = entries and self.playlist_result(entries, num_id)
+            if not result:
+                item_data = try_get(page_props, lambda x: x['ItemList']['user-post']['list'], list)
+                result = self.playlist_from_matches(item_data, playlist_id=num_id, getter=lambda m: 'tiktok:%s' % (m, ))
+            if result:
+                result['display_id'] = user_id
+                return result
+
         data = self._download_json(
             'https://m.tiktok.com/h5/share/usr/list/%s/' % user_id, user_id,
             query={'_signature': '_'})
-        entries = []
         for aweme in data['aweme_list']:
             try:
                 entry = self._extract_video(aweme)
@@ -181,4 +203,5 @@ def _real_extract(self, url):
                 continue
             entry['extractor_key'] = TikTokIE.ie_key()
             entries.append(entry)
+
         return self.playlist_result(entries, user_id)

From 70255654e60063deb7a5dd9d8f63dcd35210280b Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sat, 8 Jan 2022 11:42:43 +0400
Subject: [PATCH 092/394] Updated streamff

---
 youtube_dl/extractor/streamff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
index 8bcecdad6b5..5cf13530e3c 100644
--- a/youtube_dl/extractor/streamff.py
+++ b/youtube_dl/extractor/streamff.py
@@ -22,5 +22,5 @@ class StreamFFIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_url = "https://d32h9bi2u31gge.cloudfront.net/uploads/%s.mp4" % video_id
+        video_url = "https://streamffcdn.b-cdn.net/uploads/%s.mp4" % video_id
         return self.url_result(video_url, ie="Generic")

From 765a0fb77de0459765f7a57793b70abd2b53d35c Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sat, 15 Jan 2022 00:10:12 +0400
Subject: [PATCH 093/394] Updated streamff

---
 youtube_dl/extractor/streamff.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
index 5cf13530e3c..ebadae7c4f1 100644
--- a/youtube_dl/extractor/streamff.py
+++ b/youtube_dl/extractor/streamff.py
@@ -22,5 +22,6 @@ class StreamFFIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_url = "https://streamffcdn.b-cdn.net/uploads/%s.mp4" % video_id
+        response = self._download_json("https://streamff.com/api/videos/%s" % video_id, video_id)
+        video_url = "https://streamff.com%s" % response["videoLink"]
         return self.url_result(video_url, ie="Generic")

From 39a3c6150043318a800d62c298b8d68cca6f3200 Mon Sep 17 00:00:00 2001
From: yayorbitgum <50963144+yayorbitgum@users.noreply.github.com>
Date: Sat, 15 Jan 2022 16:49:38 -0600
Subject: [PATCH 094/394] Fixed groups() call on potentially empty regex search
 object. - https://github.com/ytdl-org/youtube-dl/issues/30521

---
 youtube_dl/extractor/myspass.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index db7ebc94ca7..39e56f7df5d 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -35,11 +35,14 @@ def _real_extract(self, url):
         title = xpath_text(metadata, 'title', fatal=True)
         video_url = xpath_text(metadata, 'url_flv', 'download url', True)
         video_id_int = int(video_id)
-        for group in re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url).groups():
-            group_int = int(group)
-            if group_int > video_id_int:
-                video_url = video_url.replace(
-                    group, compat_str(group_int // video_id_int))
+
+        regex_check = re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url)
+        if regex_check:
+            for group in regex_check.groups():
+                group_int = int(group)
+                if group_int > video_id_int:
+                    video_url = video_url.replace(
+                        group, compat_str(group_int // video_id_int))
 
         return {
             'id': video_id,

From 0945174fa4326d5bd4b04d2b6f5ed104569c5559 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jan 2022 14:17:48 +0000
Subject: [PATCH 095/394] Work-around duration parsing pending fix to
 utils.parse_duration

---
 youtube_dl/extractor/itv.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 4acfa81ad60..67a000e04b0 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -7,7 +7,7 @@
 from .brightcove import BrightcoveNewIE
 
 from ..compat import compat_str
-from ..utils import (
+from ..utils import (  # noqa: F401
     base_url,
     clean_html,
     determine_ext,
@@ -209,6 +209,13 @@ def _real_extract(self, url):
             })
         self._remove_duplicate_formats(thumbnails)
 
+        # TODO: remove this once utils.py is updated
+        def parse_duration(s):  # noqa: F811
+            from re import sub
+            from ..utils import parse_duration as utils_parse_duration
+            return utils_parse_duration(
+                sub(r':(\d{3,})$', r'.\1', s or ''))
+
         return merge_dicts({
             'id': video_id,
             'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),

From 2f6a844faf0d115d71c541a008b3c8d2cc92ef28 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Jan 2022 14:54:43 +0000
Subject: [PATCH 096/394] Improve ExtractorError with msg IV and ie constructor
 param

---
 youtube_dl/utils.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0bc6be5094b..3c8b748b46c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2348,26 +2348,28 @@ class YoutubeDLError(Exception):
 class ExtractorError(YoutubeDLError):
     """Error during info extraction."""
 
-    def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
+    def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None, ie=None):
         """ tb, if given, is the original traceback (so that it can be printed out).
         If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
         """
 
-        if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
-            expected = True
-        if video_id is not None:
-            msg = video_id + ': ' + msg
-        if cause:
-            msg += ' (caused by %r)' % cause
-        if not expected:
-            msg += bug_reports_message()
-        super(ExtractorError, self).__init__(msg)
-
+        self.msg = compat_str(msg)
         self.traceback = tb
         self.exc_info = sys.exc_info()  # preserve original exception
         self.cause = cause
         self.video_id = video_id
 
+        expected = expected or (
+            self.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError))
+        msg = ''.join((
+            '[%s] ' % ie if ie is not None else '',
+            video_id + ': ' if video_id is not None else '',
+            self.msg or 'Extractor error',
+            (' (caused by %r)' % (cause, )) if cause else '',
+            bug_reports_message() if not expected else '',
+        ))
+        super(ExtractorError, self).__init__(msg)
+
     def format_traceback(self):
         if self.traceback is None:
             return None

From cd1ffab8e17c2d6126e3bbe361c8c760ba3aefc6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Jan 2022 17:00:08 +0000
Subject: [PATCH 097/394] Work-around for yt-dlp issue 1060 (skip bad certs
 from Windows, Py>=3.7)

---
 youtube_dl/utils.py | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3c8b748b46c..6b18cb893cd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2305,12 +2305,39 @@ def formatSeconds(secs):
 
 
 def make_HTTPS_handler(params, **kwargs):
-    opts_no_check_certificate = params.get('nocheckcertificate', False)
+    opts_check_certificate = not params.get('nocheckcertificate', False)
+
+    if sys.version_info >= (3, 7) and opts_check_certificate and sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+        # From yt-dlp: work around issue in load_default_certs if there are bad certificates. See:
+        # https://github.com/yt-dlp/yt-dlp/issues/1060,
+        # https://bugs.python.org/issue35665, https://bugs.python.org/issue4531
+        # but not with MingW (no enum_certificates())
+        from types import MethodType
+
+        def _ssl_load_windows_store_certs(ssl_context, storename):
+            # Code adapted from _load_windows_store_certs in https://github.com/python/cpython/blob/main/Lib/ssl.py
+            try:
+                certs = [cert for cert, encoding, trust in ssl.enum_certificates(storename)
+                         if encoding == 'x509_asn' and (
+                             trust is True or ssl.Purpose.SERVER_AUTH.oid in trust)]
+            except PermissionError:
+                return
+            for cert in certs:
+                try:
+                    ssl_context.load_verify_locations(cadata=cert)
+                except ssl.SSLError:
+                    # no warning here, as the problem is probably a permanent unfixable feature of the platform
+                    pass
+
+        ssl.SSLContext._ssl_load_windows_store_certs = MethodType(_ssl_load_windows_store_certs, ssl.SSLContext)
+        ssl_cert_verify_mode = True
+    else:
+        ssl_cert_verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
+
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
-        if opts_no_check_certificate:
-            context.check_hostname = False
-            context.verify_mode = ssl.CERT_NONE
+        context.check_hostname = opts_check_certificate
+        context.verify_mode = ssl_cert_verify_mode
         try:
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
         except TypeError:
@@ -2322,9 +2349,7 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
-        context.verify_mode = (ssl.CERT_NONE
-                               if opts_no_check_certificate
-                               else ssl.CERT_REQUIRED)
+        context.verify_mode = ssl_cert_verify_mode
         context.set_default_verify_paths()
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 

From cad0ff54113d578eb4033ca300b9b735ebabd626 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Jan 2022 18:12:55 +0000
Subject: [PATCH 098/394] Ensure a value from determine_protocol()

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6b18cb893cd..1e5291285d7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4419,7 +4419,7 @@ def determine_protocol(info_dict):
     elif ext == 'f4m':
         return 'f4m'
 
-    return compat_urllib_parse_urlparse(url).scheme
+    return compat_urllib_parse_urlparse(url).scheme or 'http'
 
 
 def render_table(header_row, data):

From 5ed91cf9d0f514b894bda08dc40415ba96f3cb95 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Jan 2022 01:09:00 +0000
Subject: [PATCH 099/394] Simplify init_or_none(), based on yt-dlp
 9e907ebddf9c9c3084fb1b24156c9e3786ab428e

Also swallow inf, nan
---
 youtube_dl/utils.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1e5291285d7..2de3556a2e9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3694,16 +3694,11 @@ def get_method(self):
 
 
 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
-    if get_attr:
-        if v is not None:
-            v = getattr(v, get_attr, None)
-    if v == '':
-        v = None
-    if v is None:
-        return default
+    if get_attr and v is not None:
+        v = getattr(v, get_attr, None)
     try:
         return int(v) * invscale // scale
-    except (ValueError, TypeError):
+    except (ValueError, TypeError, OverflowError):
         return default
 
 

From 5f86d8a00063c79bd2f4ef761c3cc51193bdd672 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Jan 2022 20:33:04 +0000
Subject: [PATCH 100/394] Sort Chrome versions used for UAs; drop obsolete
 `Accept-Charset` header

---
 youtube_dl/utils.py | 3149 +++++++++++++++++++++----------------------
 1 file changed, 1574 insertions(+), 1575 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2de3556a2e9..f1139e61575 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -89,1595 +89,1594 @@ def register_socks_protocols():
 def random_user_agent():
     _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
     _CHROME_VERSIONS = (
-        '74.0.3729.129',
-        '76.0.3780.3',
-        '76.0.3780.2',
-        '74.0.3729.128',
-        '76.0.3780.1',
-        '76.0.3780.0',
-        '75.0.3770.15',
-        '74.0.3729.127',
-        '74.0.3729.126',
-        '76.0.3779.1',
-        '76.0.3779.0',
-        '75.0.3770.14',
-        '74.0.3729.125',
-        '76.0.3778.1',
-        '76.0.3778.0',
-        '75.0.3770.13',
-        '74.0.3729.124',
-        '74.0.3729.123',
-        '73.0.3683.121',
-        '76.0.3777.1',
-        '76.0.3777.0',
-        '75.0.3770.12',
-        '74.0.3729.122',
-        '76.0.3776.4',
-        '75.0.3770.11',
-        '74.0.3729.121',
-        '76.0.3776.3',
-        '76.0.3776.2',
-        '73.0.3683.120',
-        '74.0.3729.120',
-        '74.0.3729.119',
-        '74.0.3729.118',
-        '76.0.3776.1',
-        '76.0.3776.0',
-        '76.0.3775.5',
-        '75.0.3770.10',
-        '74.0.3729.117',
-        '76.0.3775.4',
-        '76.0.3775.3',
-        '74.0.3729.116',
-        '75.0.3770.9',
-        '76.0.3775.2',
-        '76.0.3775.1',
-        '76.0.3775.0',
-        '75.0.3770.8',
-        '74.0.3729.115',
-        '74.0.3729.114',
-        '76.0.3774.1',
-        '76.0.3774.0',
-        '75.0.3770.7',
-        '74.0.3729.113',
-        '74.0.3729.112',
-        '74.0.3729.111',
-        '76.0.3773.1',
-        '76.0.3773.0',
-        '75.0.3770.6',
-        '74.0.3729.110',
-        '74.0.3729.109',
-        '76.0.3772.1',
-        '76.0.3772.0',
-        '75.0.3770.5',
-        '74.0.3729.108',
-        '74.0.3729.107',
-        '76.0.3771.1',
-        '76.0.3771.0',
-        '75.0.3770.4',
-        '74.0.3729.106',
-        '74.0.3729.105',
-        '75.0.3770.3',
-        '74.0.3729.104',
-        '74.0.3729.103',
-        '74.0.3729.102',
-        '75.0.3770.2',
-        '74.0.3729.101',
-        '75.0.3770.1',
-        '75.0.3770.0',
-        '74.0.3729.100',
-        '75.0.3769.5',
-        '75.0.3769.4',
-        '74.0.3729.99',
-        '75.0.3769.3',
-        '75.0.3769.2',
-        '75.0.3768.6',
-        '74.0.3729.98',
-        '75.0.3769.1',
-        '75.0.3769.0',
-        '74.0.3729.97',
-        '73.0.3683.119',
-        '73.0.3683.118',
-        '74.0.3729.96',
-        '75.0.3768.5',
-        '75.0.3768.4',
-        '75.0.3768.3',
-        '75.0.3768.2',
-        '74.0.3729.95',
-        '74.0.3729.94',
-        '75.0.3768.1',
-        '75.0.3768.0',
-        '74.0.3729.93',
-        '74.0.3729.92',
-        '73.0.3683.117',
-        '74.0.3729.91',
-        '75.0.3766.3',
-        '74.0.3729.90',
-        '75.0.3767.2',
-        '75.0.3767.1',
-        '75.0.3767.0',
-        '74.0.3729.89',
-        '73.0.3683.116',
-        '75.0.3766.2',
-        '74.0.3729.88',
-        '75.0.3766.1',
-        '75.0.3766.0',
-        '74.0.3729.87',
-        '73.0.3683.115',
-        '74.0.3729.86',
-        '75.0.3765.1',
-        '75.0.3765.0',
-        '74.0.3729.85',
-        '73.0.3683.114',
-        '74.0.3729.84',
-        '75.0.3764.1',
-        '75.0.3764.0',
-        '74.0.3729.83',
-        '73.0.3683.113',
-        '75.0.3763.2',
-        '75.0.3761.4',
-        '74.0.3729.82',
-        '75.0.3763.1',
-        '75.0.3763.0',
-        '74.0.3729.81',
-        '73.0.3683.112',
-        '75.0.3762.1',
-        '75.0.3762.0',
-        '74.0.3729.80',
-        '75.0.3761.3',
-        '74.0.3729.79',
-        '73.0.3683.111',
-        '75.0.3761.2',
-        '74.0.3729.78',
-        '74.0.3729.77',
-        '75.0.3761.1',
-        '75.0.3761.0',
-        '73.0.3683.110',
-        '74.0.3729.76',
-        '74.0.3729.75',
-        '75.0.3760.0',
-        '74.0.3729.74',
-        '75.0.3759.8',
-        '75.0.3759.7',
-        '75.0.3759.6',
-        '74.0.3729.73',
-        '75.0.3759.5',
-        '74.0.3729.72',
-        '73.0.3683.109',
-        '75.0.3759.4',
-        '75.0.3759.3',
-        '74.0.3729.71',
-        '75.0.3759.2',
-        '74.0.3729.70',
-        '73.0.3683.108',
-        '74.0.3729.69',
-        '75.0.3759.1',
-        '75.0.3759.0',
-        '74.0.3729.68',
-        '73.0.3683.107',
-        '74.0.3729.67',
-        '75.0.3758.1',
-        '75.0.3758.0',
-        '74.0.3729.66',
-        '73.0.3683.106',
-        '74.0.3729.65',
-        '75.0.3757.1',
-        '75.0.3757.0',
-        '74.0.3729.64',
-        '73.0.3683.105',
-        '74.0.3729.63',
-        '75.0.3756.1',
-        '75.0.3756.0',
-        '74.0.3729.62',
-        '73.0.3683.104',
-        '75.0.3755.3',
-        '75.0.3755.2',
-        '73.0.3683.103',
-        '75.0.3755.1',
-        '75.0.3755.0',
-        '74.0.3729.61',
-        '73.0.3683.102',
-        '74.0.3729.60',
-        '75.0.3754.2',
-        '74.0.3729.59',
-        '75.0.3753.4',
-        '74.0.3729.58',
-        '75.0.3754.1',
-        '75.0.3754.0',
-        '74.0.3729.57',
-        '73.0.3683.101',
-        '75.0.3753.3',
-        '75.0.3752.2',
-        '75.0.3753.2',
-        '74.0.3729.56',
-        '75.0.3753.1',
-        '75.0.3753.0',
-        '74.0.3729.55',
-        '73.0.3683.100',
-        '74.0.3729.54',
-        '75.0.3752.1',
-        '75.0.3752.0',
-        '74.0.3729.53',
-        '73.0.3683.99',
-        '74.0.3729.52',
-        '75.0.3751.1',
-        '75.0.3751.0',
-        '74.0.3729.51',
-        '73.0.3683.98',
-        '74.0.3729.50',
-        '75.0.3750.0',
-        '74.0.3729.49',
-        '74.0.3729.48',
-        '74.0.3729.47',
-        '75.0.3749.3',
-        '74.0.3729.46',
-        '73.0.3683.97',
-        '75.0.3749.2',
-        '74.0.3729.45',
-        '75.0.3749.1',
-        '75.0.3749.0',
-        '74.0.3729.44',
-        '73.0.3683.96',
-        '74.0.3729.43',
-        '74.0.3729.42',
-        '75.0.3748.1',
-        '75.0.3748.0',
-        '74.0.3729.41',
-        '75.0.3747.1',
-        '73.0.3683.95',
-        '75.0.3746.4',
-        '74.0.3729.40',
-        '74.0.3729.39',
-        '75.0.3747.0',
-        '75.0.3746.3',
-        '75.0.3746.2',
-        '74.0.3729.38',
-        '75.0.3746.1',
-        '75.0.3746.0',
-        '74.0.3729.37',
-        '73.0.3683.94',
-        '75.0.3745.5',
-        '75.0.3745.4',
-        '75.0.3745.3',
-        '75.0.3745.2',
-        '74.0.3729.36',
-        '75.0.3745.1',
-        '75.0.3745.0',
-        '75.0.3744.2',
-        '74.0.3729.35',
-        '73.0.3683.93',
-        '74.0.3729.34',
-        '75.0.3744.1',
-        '75.0.3744.0',
-        '74.0.3729.33',
-        '73.0.3683.92',
-        '74.0.3729.32',
-        '74.0.3729.31',
-        '73.0.3683.91',
-        '75.0.3741.2',
-        '75.0.3740.5',
-        '74.0.3729.30',
-        '75.0.3741.1',
-        '75.0.3741.0',
-        '74.0.3729.29',
-        '75.0.3740.4',
-        '73.0.3683.90',
-        '74.0.3729.28',
-        '75.0.3740.3',
-        '73.0.3683.89',
-        '75.0.3740.2',
-        '74.0.3729.27',
-        '75.0.3740.1',
-        '75.0.3740.0',
-        '74.0.3729.26',
-        '73.0.3683.88',
-        '73.0.3683.87',
-        '74.0.3729.25',
-        '75.0.3739.1',
-        '75.0.3739.0',
-        '73.0.3683.86',
-        '74.0.3729.24',
-        '73.0.3683.85',
-        '75.0.3738.4',
-        '75.0.3738.3',
-        '75.0.3738.2',
-        '75.0.3738.1',
-        '75.0.3738.0',
-        '74.0.3729.23',
-        '73.0.3683.84',
-        '74.0.3729.22',
-        '74.0.3729.21',
-        '75.0.3737.1',
-        '75.0.3737.0',
-        '74.0.3729.20',
-        '73.0.3683.83',
-        '74.0.3729.19',
-        '75.0.3736.1',
-        '75.0.3736.0',
-        '74.0.3729.18',
-        '73.0.3683.82',
-        '74.0.3729.17',
-        '75.0.3735.1',
-        '75.0.3735.0',
-        '74.0.3729.16',
-        '73.0.3683.81',
-        '75.0.3734.1',
-        '75.0.3734.0',
-        '74.0.3729.15',
-        '73.0.3683.80',
-        '74.0.3729.14',
-        '75.0.3733.1',
-        '75.0.3733.0',
-        '75.0.3732.1',
-        '74.0.3729.13',
-        '74.0.3729.12',
-        '73.0.3683.79',
-        '74.0.3729.11',
-        '75.0.3732.0',
-        '74.0.3729.10',
-        '73.0.3683.78',
-        '74.0.3729.9',
-        '74.0.3729.8',
-        '74.0.3729.7',
-        '75.0.3731.3',
-        '75.0.3731.2',
-        '75.0.3731.0',
-        '74.0.3729.6',
-        '73.0.3683.77',
-        '73.0.3683.76',
-        '75.0.3730.5',
-        '75.0.3730.4',
-        '73.0.3683.75',
-        '74.0.3729.5',
-        '73.0.3683.74',
-        '75.0.3730.3',
-        '75.0.3730.2',
-        '74.0.3729.4',
-        '73.0.3683.73',
-        '73.0.3683.72',
-        '75.0.3730.1',
-        '75.0.3730.0',
-        '74.0.3729.3',
-        '73.0.3683.71',
-        '74.0.3729.2',
-        '73.0.3683.70',
-        '74.0.3729.1',
-        '74.0.3729.0',
-        '74.0.3726.4',
-        '73.0.3683.69',
-        '74.0.3726.3',
-        '74.0.3728.0',
-        '74.0.3726.2',
-        '73.0.3683.68',
-        '74.0.3726.1',
-        '74.0.3726.0',
-        '74.0.3725.4',
-        '73.0.3683.67',
-        '73.0.3683.66',
-        '74.0.3725.3',
-        '74.0.3725.2',
-        '74.0.3725.1',
-        '74.0.3724.8',
-        '74.0.3725.0',
-        '73.0.3683.65',
-        '74.0.3724.7',
-        '74.0.3724.6',
-        '74.0.3724.5',
-        '74.0.3724.4',
-        '74.0.3724.3',
-        '74.0.3724.2',
-        '74.0.3724.1',
-        '74.0.3724.0',
-        '73.0.3683.64',
-        '74.0.3723.1',
-        '74.0.3723.0',
-        '73.0.3683.63',
-        '74.0.3722.1',
-        '74.0.3722.0',
-        '73.0.3683.62',
-        '74.0.3718.9',
-        '74.0.3702.3',
-        '74.0.3721.3',
-        '74.0.3721.2',
-        '74.0.3721.1',
-        '74.0.3721.0',
-        '74.0.3720.6',
-        '73.0.3683.61',
-        '72.0.3626.122',
-        '73.0.3683.60',
-        '74.0.3720.5',
-        '72.0.3626.121',
-        '74.0.3718.8',
-        '74.0.3720.4',
-        '74.0.3720.3',
-        '74.0.3718.7',
-        '74.0.3720.2',
-        '74.0.3720.1',
-        '74.0.3720.0',
-        '74.0.3718.6',
-        '74.0.3719.5',
-        '73.0.3683.59',
-        '74.0.3718.5',
-        '74.0.3718.4',
-        '74.0.3719.4',
-        '74.0.3719.3',
-        '74.0.3719.2',
-        '74.0.3719.1',
-        '73.0.3683.58',
-        '74.0.3719.0',
-        '73.0.3683.57',
-        '73.0.3683.56',
-        '74.0.3718.3',
-        '73.0.3683.55',
-        '74.0.3718.2',
-        '74.0.3718.1',
-        '74.0.3718.0',
-        '73.0.3683.54',
-        '74.0.3717.2',
-        '73.0.3683.53',
-        '74.0.3717.1',
-        '74.0.3717.0',
-        '73.0.3683.52',
-        '74.0.3716.1',
-        '74.0.3716.0',
-        '73.0.3683.51',
-        '74.0.3715.1',
-        '74.0.3715.0',
-        '73.0.3683.50',
-        '74.0.3711.2',
-        '74.0.3714.2',
-        '74.0.3713.3',
-        '74.0.3714.1',
-        '74.0.3714.0',
-        '73.0.3683.49',
-        '74.0.3713.1',
-        '74.0.3713.0',
-        '72.0.3626.120',
-        '73.0.3683.48',
-        '74.0.3712.2',
-        '74.0.3712.1',
-        '74.0.3712.0',
-        '73.0.3683.47',
-        '72.0.3626.119',
-        '73.0.3683.46',
-        '74.0.3710.2',
-        '72.0.3626.118',
-        '74.0.3711.1',
-        '74.0.3711.0',
-        '73.0.3683.45',
-        '72.0.3626.117',
-        '74.0.3710.1',
-        '74.0.3710.0',
-        '73.0.3683.44',
-        '72.0.3626.116',
-        '74.0.3709.1',
-        '74.0.3709.0',
-        '74.0.3704.9',
-        '73.0.3683.43',
-        '72.0.3626.115',
-        '74.0.3704.8',
-        '74.0.3704.7',
-        '74.0.3708.0',
-        '74.0.3706.7',
-        '74.0.3704.6',
-        '73.0.3683.42',
-        '72.0.3626.114',
-        '74.0.3706.6',
-        '72.0.3626.113',
-        '74.0.3704.5',
-        '74.0.3706.5',
-        '74.0.3706.4',
-        '74.0.3706.3',
-        '74.0.3706.2',
-        '74.0.3706.1',
-        '74.0.3706.0',
-        '73.0.3683.41',
-        '72.0.3626.112',
-        '74.0.3705.1',
-        '74.0.3705.0',
-        '73.0.3683.40',
-        '72.0.3626.111',
-        '73.0.3683.39',
-        '74.0.3704.4',
-        '73.0.3683.38',
-        '74.0.3704.3',
-        '74.0.3704.2',
-        '74.0.3704.1',
-        '74.0.3704.0',
-        '73.0.3683.37',
-        '72.0.3626.110',
-        '72.0.3626.109',
-        '74.0.3703.3',
-        '74.0.3703.2',
-        '73.0.3683.36',
-        '74.0.3703.1',
-        '74.0.3703.0',
-        '73.0.3683.35',
-        '72.0.3626.108',
-        '74.0.3702.2',
-        '74.0.3699.3',
-        '74.0.3702.1',
-        '74.0.3702.0',
-        '73.0.3683.34',
-        '72.0.3626.107',
-        '73.0.3683.33',
-        '74.0.3701.1',
-        '74.0.3701.0',
-        '73.0.3683.32',
-        '73.0.3683.31',
-        '72.0.3626.105',
-        '74.0.3700.1',
-        '74.0.3700.0',
-        '73.0.3683.29',
-        '72.0.3626.103',
-        '74.0.3699.2',
-        '74.0.3699.1',
-        '74.0.3699.0',
-        '73.0.3683.28',
-        '72.0.3626.102',
-        '73.0.3683.27',
-        '73.0.3683.26',
-        '74.0.3698.0',
-        '74.0.3696.2',
-        '72.0.3626.101',
-        '73.0.3683.25',
-        '74.0.3696.1',
-        '74.0.3696.0',
-        '74.0.3694.8',
-        '72.0.3626.100',
-        '74.0.3694.7',
-        '74.0.3694.6',
-        '74.0.3694.5',
-        '74.0.3694.4',
-        '72.0.3626.99',
-        '72.0.3626.98',
-        '74.0.3694.3',
-        '73.0.3683.24',
-        '72.0.3626.97',
-        '72.0.3626.96',
-        '72.0.3626.95',
-        '73.0.3683.23',
-        '72.0.3626.94',
-        '73.0.3683.22',
-        '73.0.3683.21',
-        '72.0.3626.93',
-        '74.0.3694.2',
-        '72.0.3626.92',
-        '74.0.3694.1',
-        '74.0.3694.0',
-        '74.0.3693.6',
-        '73.0.3683.20',
-        '72.0.3626.91',
-        '74.0.3693.5',
-        '74.0.3693.4',
-        '74.0.3693.3',
-        '74.0.3693.2',
-        '73.0.3683.19',
-        '74.0.3693.1',
-        '74.0.3693.0',
-        '73.0.3683.18',
-        '72.0.3626.90',
-        '74.0.3692.1',
-        '74.0.3692.0',
-        '73.0.3683.17',
-        '72.0.3626.89',
-        '74.0.3687.3',
-        '74.0.3691.1',
-        '74.0.3691.0',
-        '73.0.3683.16',
-        '72.0.3626.88',
-        '72.0.3626.87',
-        '73.0.3683.15',
-        '74.0.3690.1',
-        '74.0.3690.0',
-        '73.0.3683.14',
-        '72.0.3626.86',
-        '73.0.3683.13',
-        '73.0.3683.12',
-        '74.0.3689.1',
-        '74.0.3689.0',
-        '73.0.3683.11',
-        '72.0.3626.85',
-        '73.0.3683.10',
-        '72.0.3626.84',
-        '73.0.3683.9',
-        '74.0.3688.1',
-        '74.0.3688.0',
-        '73.0.3683.8',
-        '72.0.3626.83',
-        '74.0.3687.2',
-        '74.0.3687.1',
-        '74.0.3687.0',
-        '73.0.3683.7',
-        '72.0.3626.82',
-        '74.0.3686.4',
-        '72.0.3626.81',
-        '74.0.3686.3',
-        '74.0.3686.2',
-        '74.0.3686.1',
-        '74.0.3686.0',
-        '73.0.3683.6',
-        '72.0.3626.80',
-        '74.0.3685.1',
-        '74.0.3685.0',
-        '73.0.3683.5',
-        '72.0.3626.79',
-        '74.0.3684.1',
-        '74.0.3684.0',
-        '73.0.3683.4',
-        '72.0.3626.78',
-        '72.0.3626.77',
-        '73.0.3683.3',
-        '73.0.3683.2',
-        '72.0.3626.76',
-        '73.0.3683.1',
-        '73.0.3683.0',
-        '72.0.3626.75',
-        '71.0.3578.141',
-        '73.0.3682.1',
-        '73.0.3682.0',
-        '72.0.3626.74',
-        '71.0.3578.140',
-        '73.0.3681.4',
-        '73.0.3681.3',
-        '73.0.3681.2',
-        '73.0.3681.1',
-        '73.0.3681.0',
-        '72.0.3626.73',
-        '71.0.3578.139',
-        '72.0.3626.72',
-        '72.0.3626.71',
-        '73.0.3680.1',
-        '73.0.3680.0',
-        '72.0.3626.70',
-        '71.0.3578.138',
-        '73.0.3678.2',
-        '73.0.3679.1',
-        '73.0.3679.0',
-        '72.0.3626.69',
-        '71.0.3578.137',
-        '73.0.3678.1',
-        '73.0.3678.0',
-        '71.0.3578.136',
-        '73.0.3677.1',
-        '73.0.3677.0',
-        '72.0.3626.68',
-        '72.0.3626.67',
-        '71.0.3578.135',
-        '73.0.3676.1',
-        '73.0.3676.0',
-        '73.0.3674.2',
-        '72.0.3626.66',
-        '71.0.3578.134',
-        '73.0.3674.1',
-        '73.0.3674.0',
-        '72.0.3626.65',
-        '71.0.3578.133',
-        '73.0.3673.2',
-        '73.0.3673.1',
-        '73.0.3673.0',
-        '72.0.3626.64',
-        '71.0.3578.132',
-        '72.0.3626.63',
-        '72.0.3626.62',
-        '72.0.3626.61',
-        '72.0.3626.60',
-        '73.0.3672.1',
-        '73.0.3672.0',
-        '72.0.3626.59',
-        '71.0.3578.131',
-        '73.0.3671.3',
-        '73.0.3671.2',
-        '73.0.3671.1',
-        '73.0.3671.0',
-        '72.0.3626.58',
-        '71.0.3578.130',
-        '73.0.3670.1',
-        '73.0.3670.0',
-        '72.0.3626.57',
-        '71.0.3578.129',
-        '73.0.3669.1',
-        '73.0.3669.0',
-        '72.0.3626.56',
-        '71.0.3578.128',
-        '73.0.3668.2',
-        '73.0.3668.1',
-        '73.0.3668.0',
-        '72.0.3626.55',
-        '71.0.3578.127',
-        '73.0.3667.2',
-        '73.0.3667.1',
-        '73.0.3667.0',
-        '72.0.3626.54',
-        '71.0.3578.126',
-        '73.0.3666.1',
-        '73.0.3666.0',
-        '72.0.3626.53',
-        '71.0.3578.125',
-        '73.0.3665.4',
-        '73.0.3665.3',
-        '72.0.3626.52',
-        '73.0.3665.2',
-        '73.0.3664.4',
-        '73.0.3665.1',
-        '73.0.3665.0',
-        '72.0.3626.51',
-        '71.0.3578.124',
-        '72.0.3626.50',
-        '73.0.3664.3',
-        '73.0.3664.2',
-        '73.0.3664.1',
-        '73.0.3664.0',
-        '73.0.3663.2',
-        '72.0.3626.49',
-        '71.0.3578.123',
-        '73.0.3663.1',
-        '73.0.3663.0',
-        '72.0.3626.48',
-        '71.0.3578.122',
-        '73.0.3662.1',
-        '73.0.3662.0',
-        '72.0.3626.47',
-        '71.0.3578.121',
-        '73.0.3661.1',
-        '72.0.3626.46',
-        '73.0.3661.0',
-        '72.0.3626.45',
-        '71.0.3578.120',
-        '73.0.3660.2',
-        '73.0.3660.1',
-        '73.0.3660.0',
-        '72.0.3626.44',
-        '71.0.3578.119',
-        '73.0.3659.1',
-        '73.0.3659.0',
-        '72.0.3626.43',
-        '71.0.3578.118',
-        '73.0.3658.1',
-        '73.0.3658.0',
-        '72.0.3626.42',
-        '71.0.3578.117',
-        '73.0.3657.1',
-        '73.0.3657.0',
-        '72.0.3626.41',
-        '71.0.3578.116',
-        '73.0.3656.1',
-        '73.0.3656.0',
-        '72.0.3626.40',
-        '71.0.3578.115',
-        '73.0.3655.1',
-        '73.0.3655.0',
-        '72.0.3626.39',
-        '71.0.3578.114',
-        '73.0.3654.1',
-        '73.0.3654.0',
-        '72.0.3626.38',
-        '71.0.3578.113',
-        '73.0.3653.1',
-        '73.0.3653.0',
-        '72.0.3626.37',
-        '71.0.3578.112',
-        '73.0.3652.1',
-        '73.0.3652.0',
-        '72.0.3626.36',
-        '71.0.3578.111',
-        '73.0.3651.1',
-        '73.0.3651.0',
-        '72.0.3626.35',
-        '71.0.3578.110',
-        '73.0.3650.1',
-        '73.0.3650.0',
-        '72.0.3626.34',
-        '71.0.3578.109',
-        '73.0.3649.1',
-        '73.0.3649.0',
-        '72.0.3626.33',
-        '71.0.3578.108',
-        '73.0.3648.2',
-        '73.0.3648.1',
-        '73.0.3648.0',
-        '72.0.3626.32',
-        '71.0.3578.107',
-        '73.0.3647.2',
-        '73.0.3647.1',
-        '73.0.3647.0',
-        '72.0.3626.31',
-        '71.0.3578.106',
-        '73.0.3635.3',
-        '73.0.3646.2',
-        '73.0.3646.1',
-        '73.0.3646.0',
-        '72.0.3626.30',
-        '71.0.3578.105',
-        '72.0.3626.29',
-        '73.0.3645.2',
-        '73.0.3645.1',
-        '73.0.3645.0',
-        '72.0.3626.28',
-        '71.0.3578.104',
-        '72.0.3626.27',
-        '72.0.3626.26',
-        '72.0.3626.25',
-        '72.0.3626.24',
-        '73.0.3644.0',
-        '73.0.3643.2',
-        '72.0.3626.23',
-        '71.0.3578.103',
-        '73.0.3643.1',
-        '73.0.3643.0',
-        '72.0.3626.22',
-        '71.0.3578.102',
-        '73.0.3642.1',
-        '73.0.3642.0',
-        '72.0.3626.21',
-        '71.0.3578.101',
-        '73.0.3641.1',
-        '73.0.3641.0',
-        '72.0.3626.20',
-        '71.0.3578.100',
-        '72.0.3626.19',
-        '73.0.3640.1',
-        '73.0.3640.0',
-        '72.0.3626.18',
-        '73.0.3639.1',
-        '71.0.3578.99',
-        '73.0.3639.0',
-        '72.0.3626.17',
-        '73.0.3638.2',
-        '72.0.3626.16',
-        '73.0.3638.1',
-        '73.0.3638.0',
-        '72.0.3626.15',
-        '71.0.3578.98',
-        '73.0.3635.2',
-        '71.0.3578.97',
-        '73.0.3637.1',
-        '73.0.3637.0',
-        '72.0.3626.14',
-        '71.0.3578.96',
-        '71.0.3578.95',
-        '72.0.3626.13',
-        '71.0.3578.94',
-        '73.0.3636.2',
-        '71.0.3578.93',
-        '73.0.3636.1',
-        '73.0.3636.0',
-        '72.0.3626.12',
-        '71.0.3578.92',
-        '73.0.3635.1',
-        '73.0.3635.0',
-        '72.0.3626.11',
-        '71.0.3578.91',
-        '73.0.3634.2',
-        '73.0.3634.1',
-        '73.0.3634.0',
-        '72.0.3626.10',
-        '71.0.3578.90',
-        '71.0.3578.89',
-        '73.0.3633.2',
-        '73.0.3633.1',
-        '73.0.3633.0',
-        '72.0.3610.4',
-        '72.0.3626.9',
-        '71.0.3578.88',
-        '73.0.3632.5',
-        '73.0.3632.4',
-        '73.0.3632.3',
-        '73.0.3632.2',
-        '73.0.3632.1',
-        '73.0.3632.0',
-        '72.0.3626.8',
-        '71.0.3578.87',
-        '73.0.3631.2',
-        '73.0.3631.1',
-        '73.0.3631.0',
-        '72.0.3626.7',
-        '71.0.3578.86',
-        '72.0.3626.6',
-        '73.0.3630.1',
-        '73.0.3630.0',
-        '72.0.3626.5',
-        '71.0.3578.85',
-        '72.0.3626.4',
-        '73.0.3628.3',
-        '73.0.3628.2',
-        '73.0.3629.1',
-        '73.0.3629.0',
-        '72.0.3626.3',
-        '71.0.3578.84',
-        '73.0.3628.1',
-        '73.0.3628.0',
-        '71.0.3578.83',
-        '73.0.3627.1',
-        '73.0.3627.0',
-        '72.0.3626.2',
-        '71.0.3578.82',
-        '71.0.3578.81',
-        '71.0.3578.80',
-        '72.0.3626.1',
-        '72.0.3626.0',
-        '71.0.3578.79',
-        '70.0.3538.124',
-        '71.0.3578.78',
-        '72.0.3623.4',
-        '72.0.3625.2',
-        '72.0.3625.1',
-        '72.0.3625.0',
-        '71.0.3578.77',
-        '70.0.3538.123',
-        '72.0.3624.4',
-        '72.0.3624.3',
-        '72.0.3624.2',
-        '71.0.3578.76',
-        '72.0.3624.1',
-        '72.0.3624.0',
-        '72.0.3623.3',
-        '71.0.3578.75',
-        '70.0.3538.122',
-        '71.0.3578.74',
-        '72.0.3623.2',
-        '72.0.3610.3',
-        '72.0.3623.1',
-        '72.0.3623.0',
-        '72.0.3622.3',
-        '72.0.3622.2',
-        '71.0.3578.73',
-        '70.0.3538.121',
-        '72.0.3622.1',
-        '72.0.3622.0',
-        '71.0.3578.72',
-        '70.0.3538.120',
-        '72.0.3621.1',
-        '72.0.3621.0',
-        '71.0.3578.71',
-        '70.0.3538.119',
-        '72.0.3620.1',
-        '72.0.3620.0',
-        '71.0.3578.70',
-        '70.0.3538.118',
-        '71.0.3578.69',
-        '72.0.3619.1',
-        '72.0.3619.0',
-        '71.0.3578.68',
-        '70.0.3538.117',
-        '71.0.3578.67',
-        '72.0.3618.1',
-        '72.0.3618.0',
-        '71.0.3578.66',
-        '70.0.3538.116',
-        '72.0.3617.1',
-        '72.0.3617.0',
-        '71.0.3578.65',
-        '70.0.3538.115',
-        '72.0.3602.3',
-        '71.0.3578.64',
-        '72.0.3616.1',
-        '72.0.3616.0',
-        '71.0.3578.63',
-        '70.0.3538.114',
-        '71.0.3578.62',
-        '72.0.3615.1',
-        '72.0.3615.0',
-        '71.0.3578.61',
-        '70.0.3538.113',
-        '72.0.3614.1',
-        '72.0.3614.0',
-        '71.0.3578.60',
-        '70.0.3538.112',
-        '72.0.3613.1',
-        '72.0.3613.0',
-        '71.0.3578.59',
-        '70.0.3538.111',
-        '72.0.3612.2',
-        '72.0.3612.1',
-        '72.0.3612.0',
-        '70.0.3538.110',
-        '71.0.3578.58',
-        '70.0.3538.109',
-        '72.0.3611.2',
-        '72.0.3611.1',
-        '72.0.3611.0',
-        '71.0.3578.57',
-        '70.0.3538.108',
-        '72.0.3610.2',
-        '71.0.3578.56',
-        '71.0.3578.55',
-        '72.0.3610.1',
-        '72.0.3610.0',
-        '71.0.3578.54',
-        '70.0.3538.107',
-        '71.0.3578.53',
-        '72.0.3609.3',
-        '71.0.3578.52',
-        '72.0.3609.2',
-        '71.0.3578.51',
-        '72.0.3608.5',
-        '72.0.3609.1',
-        '72.0.3609.0',
-        '71.0.3578.50',
-        '70.0.3538.106',
-        '72.0.3608.4',
-        '72.0.3608.3',
-        '72.0.3608.2',
-        '71.0.3578.49',
-        '72.0.3608.1',
-        '72.0.3608.0',
-        '70.0.3538.105',
-        '71.0.3578.48',
-        '72.0.3607.1',
-        '72.0.3607.0',
-        '71.0.3578.47',
-        '70.0.3538.104',
-        '72.0.3606.2',
-        '72.0.3606.1',
-        '72.0.3606.0',
-        '71.0.3578.46',
-        '70.0.3538.103',
-        '70.0.3538.102',
-        '72.0.3605.3',
-        '72.0.3605.2',
-        '72.0.3605.1',
-        '72.0.3605.0',
-        '71.0.3578.45',
-        '70.0.3538.101',
-        '71.0.3578.44',
-        '71.0.3578.43',
-        '70.0.3538.100',
-        '70.0.3538.99',
-        '71.0.3578.42',
-        '72.0.3604.1',
-        '72.0.3604.0',
-        '71.0.3578.41',
-        '70.0.3538.98',
-        '71.0.3578.40',
-        '72.0.3603.2',
-        '72.0.3603.1',
-        '72.0.3603.0',
-        '71.0.3578.39',
-        '70.0.3538.97',
-        '72.0.3602.2',
-        '71.0.3578.38',
-        '71.0.3578.37',
-        '72.0.3602.1',
-        '72.0.3602.0',
-        '71.0.3578.36',
-        '70.0.3538.96',
-        '72.0.3601.1',
-        '72.0.3601.0',
-        '71.0.3578.35',
-        '70.0.3538.95',
-        '72.0.3600.1',
-        '72.0.3600.0',
-        '71.0.3578.34',
-        '70.0.3538.94',
-        '72.0.3599.3',
-        '72.0.3599.2',
-        '72.0.3599.1',
-        '72.0.3599.0',
-        '71.0.3578.33',
-        '70.0.3538.93',
-        '72.0.3598.1',
-        '72.0.3598.0',
-        '71.0.3578.32',
-        '70.0.3538.87',
-        '72.0.3597.1',
-        '72.0.3597.0',
-        '72.0.3596.2',
-        '71.0.3578.31',
-        '70.0.3538.86',
-        '71.0.3578.30',
-        '71.0.3578.29',
-        '72.0.3596.1',
-        '72.0.3596.0',
-        '71.0.3578.28',
-        '70.0.3538.85',
-        '72.0.3595.2',
-        '72.0.3591.3',
-        '72.0.3595.1',
-        '72.0.3595.0',
-        '71.0.3578.27',
-        '70.0.3538.84',
-        '72.0.3594.1',
-        '72.0.3594.0',
-        '71.0.3578.26',
-        '70.0.3538.83',
-        '72.0.3593.2',
-        '72.0.3593.1',
-        '72.0.3593.0',
-        '71.0.3578.25',
-        '70.0.3538.82',
-        '72.0.3589.3',
-        '72.0.3592.2',
-        '72.0.3592.1',
-        '72.0.3592.0',
-        '71.0.3578.24',
-        '72.0.3589.2',
-        '70.0.3538.81',
-        '70.0.3538.80',
-        '72.0.3591.2',
-        '72.0.3591.1',
-        '72.0.3591.0',
-        '71.0.3578.23',
-        '70.0.3538.79',
-        '71.0.3578.22',
-        '72.0.3590.1',
-        '72.0.3590.0',
-        '71.0.3578.21',
-        '70.0.3538.78',
-        '70.0.3538.77',
-        '72.0.3589.1',
-        '72.0.3589.0',
-        '71.0.3578.20',
-        '70.0.3538.76',
-        '71.0.3578.19',
-        '70.0.3538.75',
-        '72.0.3588.1',
-        '72.0.3588.0',
-        '71.0.3578.18',
-        '70.0.3538.74',
-        '72.0.3586.2',
-        '72.0.3587.0',
-        '71.0.3578.17',
-        '70.0.3538.73',
-        '72.0.3586.1',
-        '72.0.3586.0',
-        '71.0.3578.16',
-        '70.0.3538.72',
-        '72.0.3585.1',
-        '72.0.3585.0',
-        '71.0.3578.15',
-        '70.0.3538.71',
-        '71.0.3578.14',
-        '72.0.3584.1',
-        '72.0.3584.0',
-        '71.0.3578.13',
-        '70.0.3538.70',
-        '72.0.3583.2',
-        '71.0.3578.12',
-        '72.0.3583.1',
-        '72.0.3583.0',
-        '71.0.3578.11',
-        '70.0.3538.69',
-        '71.0.3578.10',
-        '72.0.3582.0',
-        '72.0.3581.4',
-        '71.0.3578.9',
-        '70.0.3538.67',
-        '72.0.3581.3',
-        '72.0.3581.2',
-        '72.0.3581.1',
-        '72.0.3581.0',
-        '71.0.3578.8',
-        '70.0.3538.66',
-        '72.0.3580.1',
-        '72.0.3580.0',
-        '71.0.3578.7',
-        '70.0.3538.65',
-        '71.0.3578.6',
-        '72.0.3579.1',
-        '72.0.3579.0',
-        '71.0.3578.5',
-        '70.0.3538.64',
-        '71.0.3578.4',
-        '71.0.3578.3',
-        '71.0.3578.2',
-        '71.0.3578.1',
-        '71.0.3578.0',
-        '70.0.3538.63',
-        '69.0.3497.128',
-        '70.0.3538.62',
-        '70.0.3538.61',
-        '70.0.3538.60',
-        '70.0.3538.59',
-        '71.0.3577.1',
-        '71.0.3577.0',
-        '70.0.3538.58',
-        '69.0.3497.127',
-        '71.0.3576.2',
-        '71.0.3576.1',
-        '71.0.3576.0',
-        '70.0.3538.57',
-        '70.0.3538.56',
-        '71.0.3575.2',
-        '70.0.3538.55',
-        '69.0.3497.126',
-        '70.0.3538.54',
-        '71.0.3575.1',
-        '71.0.3575.0',
-        '71.0.3574.1',
-        '71.0.3574.0',
-        '70.0.3538.53',
-        '69.0.3497.125',
-        '70.0.3538.52',
-        '71.0.3573.1',
-        '71.0.3573.0',
-        '70.0.3538.51',
-        '69.0.3497.124',
-        '71.0.3572.1',
-        '71.0.3572.0',
-        '70.0.3538.50',
-        '69.0.3497.123',
-        '71.0.3571.2',
-        '70.0.3538.49',
-        '69.0.3497.122',
-        '71.0.3571.1',
-        '71.0.3571.0',
-        '70.0.3538.48',
-        '69.0.3497.121',
-        '71.0.3570.1',
-        '71.0.3570.0',
-        '70.0.3538.47',
-        '69.0.3497.120',
-        '71.0.3568.2',
-        '71.0.3569.1',
-        '71.0.3569.0',
-        '70.0.3538.46',
-        '69.0.3497.119',
-        '70.0.3538.45',
-        '71.0.3568.1',
-        '71.0.3568.0',
-        '70.0.3538.44',
-        '69.0.3497.118',
-        '70.0.3538.43',
-        '70.0.3538.42',
-        '71.0.3567.1',
-        '71.0.3567.0',
-        '70.0.3538.41',
-        '69.0.3497.117',
-        '71.0.3566.1',
-        '71.0.3566.0',
-        '70.0.3538.40',
-        '69.0.3497.116',
-        '71.0.3565.1',
-        '71.0.3565.0',
-        '70.0.3538.39',
-        '69.0.3497.115',
-        '71.0.3564.1',
-        '71.0.3564.0',
-        '70.0.3538.38',
-        '69.0.3497.114',
-        '71.0.3563.0',
-        '71.0.3562.2',
-        '70.0.3538.37',
-        '69.0.3497.113',
-        '70.0.3538.36',
-        '70.0.3538.35',
-        '71.0.3562.1',
-        '71.0.3562.0',
-        '70.0.3538.34',
-        '69.0.3497.112',
-        '70.0.3538.33',
-        '71.0.3561.1',
-        '71.0.3561.0',
-        '70.0.3538.32',
-        '69.0.3497.111',
-        '71.0.3559.6',
-        '71.0.3560.1',
-        '71.0.3560.0',
-        '71.0.3559.5',
-        '71.0.3559.4',
-        '70.0.3538.31',
-        '69.0.3497.110',
-        '71.0.3559.3',
-        '70.0.3538.30',
-        '69.0.3497.109',
-        '71.0.3559.2',
-        '71.0.3559.1',
-        '71.0.3559.0',
-        '70.0.3538.29',
-        '69.0.3497.108',
-        '71.0.3558.2',
-        '71.0.3558.1',
-        '71.0.3558.0',
-        '70.0.3538.28',
-        '69.0.3497.107',
-        '71.0.3557.2',
-        '71.0.3557.1',
-        '71.0.3557.0',
-        '70.0.3538.27',
-        '69.0.3497.106',
-        '71.0.3554.4',
-        '70.0.3538.26',
-        '71.0.3556.1',
-        '71.0.3556.0',
-        '70.0.3538.25',
-        '71.0.3554.3',
-        '69.0.3497.105',
-        '71.0.3554.2',
-        '70.0.3538.24',
-        '69.0.3497.104',
-        '71.0.3555.2',
-        '70.0.3538.23',
-        '71.0.3555.1',
-        '71.0.3555.0',
-        '70.0.3538.22',
-        '69.0.3497.103',
-        '71.0.3554.1',
-        '71.0.3554.0',
-        '70.0.3538.21',
-        '69.0.3497.102',
-        '71.0.3553.3',
-        '70.0.3538.20',
-        '69.0.3497.101',
-        '71.0.3553.2',
-        '69.0.3497.100',
-        '71.0.3553.1',
-        '71.0.3553.0',
-        '70.0.3538.19',
-        '69.0.3497.99',
-        '69.0.3497.98',
-        '69.0.3497.97',
-        '71.0.3552.6',
-        '71.0.3552.5',
-        '71.0.3552.4',
-        '71.0.3552.3',
-        '71.0.3552.2',
-        '71.0.3552.1',
-        '71.0.3552.0',
-        '70.0.3538.18',
-        '69.0.3497.96',
-        '71.0.3551.3',
-        '71.0.3551.2',
-        '71.0.3551.1',
-        '71.0.3551.0',
-        '70.0.3538.17',
-        '69.0.3497.95',
-        '71.0.3550.3',
-        '71.0.3550.2',
-        '71.0.3550.1',
-        '71.0.3550.0',
-        '70.0.3538.16',
-        '69.0.3497.94',
-        '71.0.3549.1',
-        '71.0.3549.0',
-        '70.0.3538.15',
-        '69.0.3497.93',
-        '69.0.3497.92',
-        '71.0.3548.1',
-        '71.0.3548.0',
-        '70.0.3538.14',
-        '69.0.3497.91',
-        '71.0.3547.1',
-        '71.0.3547.0',
-        '70.0.3538.13',
-        '69.0.3497.90',
-        '71.0.3546.2',
-        '69.0.3497.89',
-        '71.0.3546.1',
-        '71.0.3546.0',
-        '70.0.3538.12',
-        '69.0.3497.88',
-        '71.0.3545.4',
-        '71.0.3545.3',
-        '71.0.3545.2',
-        '71.0.3545.1',
-        '71.0.3545.0',
-        '70.0.3538.11',
-        '69.0.3497.87',
-        '71.0.3544.5',
-        '71.0.3544.4',
-        '71.0.3544.3',
-        '71.0.3544.2',
-        '71.0.3544.1',
-        '71.0.3544.0',
-        '69.0.3497.86',
-        '70.0.3538.10',
-        '69.0.3497.85',
-        '70.0.3538.9',
-        '69.0.3497.84',
-        '71.0.3543.4',
-        '70.0.3538.8',
-        '71.0.3543.3',
-        '71.0.3543.2',
-        '71.0.3543.1',
-        '71.0.3543.0',
-        '70.0.3538.7',
-        '69.0.3497.83',
-        '71.0.3542.2',
-        '71.0.3542.1',
-        '71.0.3542.0',
-        '70.0.3538.6',
-        '69.0.3497.82',
-        '69.0.3497.81',
-        '71.0.3541.1',
-        '71.0.3541.0',
-        '70.0.3538.5',
-        '69.0.3497.80',
-        '71.0.3540.1',
-        '71.0.3540.0',
-        '70.0.3538.4',
-        '69.0.3497.79',
-        '70.0.3538.3',
-        '71.0.3539.1',
-        '71.0.3539.0',
-        '69.0.3497.78',
-        '68.0.3440.134',
-        '69.0.3497.77',
-        '70.0.3538.2',
-        '70.0.3538.1',
-        '70.0.3538.0',
-        '69.0.3497.76',
-        '68.0.3440.133',
-        '69.0.3497.75',
-        '70.0.3537.2',
-        '70.0.3537.1',
-        '70.0.3537.0',
-        '69.0.3497.74',
-        '68.0.3440.132',
-        '70.0.3536.0',
-        '70.0.3535.5',
-        '70.0.3535.4',
-        '70.0.3535.3',
-        '69.0.3497.73',
-        '68.0.3440.131',
-        '70.0.3532.8',
-        '70.0.3532.7',
-        '69.0.3497.72',
-        '69.0.3497.71',
-        '70.0.3535.2',
-        '70.0.3535.1',
-        '70.0.3535.0',
-        '69.0.3497.70',
-        '68.0.3440.130',
-        '69.0.3497.69',
-        '68.0.3440.129',
-        '70.0.3534.4',
-        '70.0.3534.3',
-        '70.0.3534.2',
-        '70.0.3534.1',
-        '70.0.3534.0',
-        '69.0.3497.68',
-        '68.0.3440.128',
-        '70.0.3533.2',
-        '70.0.3533.1',
-        '70.0.3533.0',
-        '69.0.3497.67',
-        '68.0.3440.127',
-        '70.0.3532.6',
-        '70.0.3532.5',
-        '70.0.3532.4',
-        '69.0.3497.66',
-        '68.0.3440.126',
-        '70.0.3532.3',
-        '70.0.3532.2',
-        '70.0.3532.1',
-        '69.0.3497.60',
-        '69.0.3497.65',
-        '69.0.3497.64',
-        '70.0.3532.0',
-        '70.0.3531.0',
-        '70.0.3530.4',
-        '70.0.3530.3',
-        '70.0.3530.2',
-        '69.0.3497.58',
-        '68.0.3440.125',
-        '69.0.3497.57',
-        '69.0.3497.56',
-        '69.0.3497.55',
-        '69.0.3497.54',
-        '70.0.3530.1',
-        '70.0.3530.0',
-        '69.0.3497.53',
-        '68.0.3440.124',
-        '69.0.3497.52',
-        '70.0.3529.3',
-        '70.0.3529.2',
-        '70.0.3529.1',
-        '70.0.3529.0',
-        '69.0.3497.51',
-        '70.0.3528.4',
-        '68.0.3440.123',
-        '70.0.3528.3',
-        '70.0.3528.2',
-        '70.0.3528.1',
-        '70.0.3528.0',
-        '69.0.3497.50',
-        '68.0.3440.122',
-        '70.0.3527.1',
-        '70.0.3527.0',
-        '69.0.3497.49',
-        '68.0.3440.121',
-        '70.0.3526.1',
-        '70.0.3526.0',
-        '68.0.3440.120',
-        '69.0.3497.48',
-        '69.0.3497.47',
-        '68.0.3440.119',
-        '68.0.3440.118',
-        '70.0.3525.5',
-        '70.0.3525.4',
-        '70.0.3525.3',
-        '68.0.3440.117',
-        '69.0.3497.46',
-        '70.0.3525.2',
-        '70.0.3525.1',
-        '70.0.3525.0',
-        '69.0.3497.45',
-        '68.0.3440.116',
-        '70.0.3524.4',
-        '70.0.3524.3',
-        '69.0.3497.44',
-        '70.0.3524.2',
-        '70.0.3524.1',
-        '70.0.3524.0',
-        '70.0.3523.2',
-        '69.0.3497.43',
-        '68.0.3440.115',
-        '70.0.3505.9',
-        '69.0.3497.42',
-        '70.0.3505.8',
-        '70.0.3523.1',
-        '70.0.3523.0',
-        '69.0.3497.41',
-        '68.0.3440.114',
-        '70.0.3505.7',
-        '69.0.3497.40',
-        '70.0.3522.1',
-        '70.0.3522.0',
-        '70.0.3521.2',
-        '69.0.3497.39',
-        '68.0.3440.113',
-        '70.0.3505.6',
-        '70.0.3521.1',
-        '70.0.3521.0',
-        '69.0.3497.38',
-        '68.0.3440.112',
-        '70.0.3520.1',
-        '70.0.3520.0',
-        '69.0.3497.37',
-        '68.0.3440.111',
-        '70.0.3519.3',
-        '70.0.3519.2',
-        '70.0.3519.1',
-        '70.0.3519.0',
-        '69.0.3497.36',
-        '68.0.3440.110',
-        '70.0.3518.1',
-        '70.0.3518.0',
-        '69.0.3497.35',
-        '69.0.3497.34',
-        '68.0.3440.109',
-        '70.0.3517.1',
-        '70.0.3517.0',
-        '69.0.3497.33',
-        '68.0.3440.108',
-        '69.0.3497.32',
-        '70.0.3516.3',
-        '70.0.3516.2',
-        '70.0.3516.1',
-        '70.0.3516.0',
-        '69.0.3497.31',
-        '68.0.3440.107',
-        '70.0.3515.4',
-        '68.0.3440.106',
-        '70.0.3515.3',
-        '70.0.3515.2',
-        '70.0.3515.1',
-        '70.0.3515.0',
-        '69.0.3497.30',
-        '68.0.3440.105',
+        '68.0.3440.103',
         '68.0.3440.104',
-        '70.0.3514.2',
-        '70.0.3514.1',
-        '70.0.3514.0',
+        '68.0.3440.105',
+        '68.0.3440.106',
+        '68.0.3440.107',
+        '68.0.3440.108',
+        '68.0.3440.109',
+        '68.0.3440.110',
+        '68.0.3440.111',
+        '68.0.3440.112',
+        '68.0.3440.113',
+        '68.0.3440.114',
+        '68.0.3440.115',
+        '68.0.3440.116',
+        '68.0.3440.117',
+        '68.0.3440.118',
+        '68.0.3440.119',
+        '68.0.3440.120',
+        '68.0.3440.121',
+        '68.0.3440.122',
+        '68.0.3440.123',
+        '68.0.3440.124',
+        '68.0.3440.125',
+        '68.0.3440.126',
+        '68.0.3440.127',
+        '68.0.3440.128',
+        '68.0.3440.129',
+        '68.0.3440.130',
+        '68.0.3440.131',
+        '68.0.3440.132',
+        '68.0.3440.133',
+        '68.0.3440.134',
+        '69.0.3497.100',
+        '69.0.3497.101',
+        '69.0.3497.102',
+        '69.0.3497.103',
+        '69.0.3497.104',
+        '69.0.3497.105',
+        '69.0.3497.106',
+        '69.0.3497.107',
+        '69.0.3497.108',
+        '69.0.3497.109',
+        '69.0.3497.110',
+        '69.0.3497.111',
+        '69.0.3497.112',
+        '69.0.3497.113',
+        '69.0.3497.114',
+        '69.0.3497.115',
+        '69.0.3497.116',
+        '69.0.3497.117',
+        '69.0.3497.118',
+        '69.0.3497.119',
+        '69.0.3497.120',
+        '69.0.3497.121',
+        '69.0.3497.122',
+        '69.0.3497.123',
+        '69.0.3497.124',
+        '69.0.3497.125',
+        '69.0.3497.126',
+        '69.0.3497.127',
+        '69.0.3497.128',
+        '69.0.3497.28',
         '69.0.3497.29',
-        '68.0.3440.103',
-        '70.0.3513.1',
+        '69.0.3497.30',
+        '69.0.3497.31',
+        '69.0.3497.32',
+        '69.0.3497.33',
+        '69.0.3497.34',
+        '69.0.3497.35',
+        '69.0.3497.36',
+        '69.0.3497.37',
+        '69.0.3497.38',
+        '69.0.3497.39',
+        '69.0.3497.40',
+        '69.0.3497.41',
+        '69.0.3497.42',
+        '69.0.3497.43',
+        '69.0.3497.44',
+        '69.0.3497.45',
+        '69.0.3497.46',
+        '69.0.3497.47',
+        '69.0.3497.48',
+        '69.0.3497.49',
+        '69.0.3497.50',
+        '69.0.3497.51',
+        '69.0.3497.52',
+        '69.0.3497.53',
+        '69.0.3497.54',
+        '69.0.3497.55',
+        '69.0.3497.56',
+        '69.0.3497.57',
+        '69.0.3497.58',
+        '69.0.3497.60',
+        '69.0.3497.64',
+        '69.0.3497.65',
+        '69.0.3497.66',
+        '69.0.3497.67',
+        '69.0.3497.68',
+        '69.0.3497.69',
+        '69.0.3497.70',
+        '69.0.3497.71',
+        '69.0.3497.72',
+        '69.0.3497.73',
+        '69.0.3497.74',
+        '69.0.3497.75',
+        '69.0.3497.76',
+        '69.0.3497.77',
+        '69.0.3497.78',
+        '69.0.3497.79',
+        '69.0.3497.80',
+        '69.0.3497.81',
+        '69.0.3497.82',
+        '69.0.3497.83',
+        '69.0.3497.84',
+        '69.0.3497.85',
+        '69.0.3497.86',
+        '69.0.3497.87',
+        '69.0.3497.88',
+        '69.0.3497.89',
+        '69.0.3497.90',
+        '69.0.3497.91',
+        '69.0.3497.92',
+        '69.0.3497.93',
+        '69.0.3497.94',
+        '69.0.3497.95',
+        '69.0.3497.96',
+        '69.0.3497.97',
+        '69.0.3497.98',
+        '69.0.3497.99',
+        '70.0.3505.6',
+        '70.0.3505.7',
+        '70.0.3505.8',
+        '70.0.3505.9',
         '70.0.3513.0',
-        '69.0.3497.28',
+        '70.0.3513.1',
+        '70.0.3514.0',
+        '70.0.3514.1',
+        '70.0.3514.2',
+        '70.0.3515.0',
+        '70.0.3515.1',
+        '70.0.3515.2',
+        '70.0.3515.3',
+        '70.0.3515.4',
+        '70.0.3516.0',
+        '70.0.3516.1',
+        '70.0.3516.2',
+        '70.0.3516.3',
+        '70.0.3517.0',
+        '70.0.3517.1',
+        '70.0.3518.0',
+        '70.0.3518.1',
+        '70.0.3519.0',
+        '70.0.3519.1',
+        '70.0.3519.2',
+        '70.0.3519.3',
+        '70.0.3520.0',
+        '70.0.3520.1',
+        '70.0.3521.0',
+        '70.0.3521.1',
+        '70.0.3521.2',
+        '70.0.3522.0',
+        '70.0.3522.1',
+        '70.0.3523.0',
+        '70.0.3523.1',
+        '70.0.3523.2',
+        '70.0.3524.0',
+        '70.0.3524.1',
+        '70.0.3524.2',
+        '70.0.3524.3',
+        '70.0.3524.4',
+        '70.0.3525.0',
+        '70.0.3525.1',
+        '70.0.3525.2',
+        '70.0.3525.3',
+        '70.0.3525.4',
+        '70.0.3525.5',
+        '70.0.3526.0',
+        '70.0.3526.1',
+        '70.0.3527.0',
+        '70.0.3527.1',
+        '70.0.3528.0',
+        '70.0.3528.1',
+        '70.0.3528.2',
+        '70.0.3528.3',
+        '70.0.3528.4',
+        '70.0.3529.0',
+        '70.0.3529.1',
+        '70.0.3529.2',
+        '70.0.3529.3',
+        '70.0.3530.0',
+        '70.0.3530.1',
+        '70.0.3530.2',
+        '70.0.3530.3',
+        '70.0.3530.4',
+        '70.0.3531.0',
+        '70.0.3532.0',
+        '70.0.3532.1',
+        '70.0.3532.2',
+        '70.0.3532.3',
+        '70.0.3532.4',
+        '70.0.3532.5',
+        '70.0.3532.6',
+        '70.0.3532.7',
+        '70.0.3532.8',
+        '70.0.3533.0',
+        '70.0.3533.1',
+        '70.0.3533.2',
+        '70.0.3534.0',
+        '70.0.3534.1',
+        '70.0.3534.2',
+        '70.0.3534.3',
+        '70.0.3534.4',
+        '70.0.3535.0',
+        '70.0.3535.1',
+        '70.0.3535.2',
+        '70.0.3535.3',
+        '70.0.3535.4',
+        '70.0.3535.5',
+        '70.0.3536.0',
+        '70.0.3537.0',
+        '70.0.3537.1',
+        '70.0.3537.2',
+        '70.0.3538.0',
+        '70.0.3538.1',
+        '70.0.3538.10',
+        '70.0.3538.100',
+        '70.0.3538.101',
+        '70.0.3538.102',
+        '70.0.3538.103',
+        '70.0.3538.104',
+        '70.0.3538.105',
+        '70.0.3538.106',
+        '70.0.3538.107',
+        '70.0.3538.108',
+        '70.0.3538.109',
+        '70.0.3538.11',
+        '70.0.3538.110',
+        '70.0.3538.111',
+        '70.0.3538.112',
+        '70.0.3538.113',
+        '70.0.3538.114',
+        '70.0.3538.115',
+        '70.0.3538.116',
+        '70.0.3538.117',
+        '70.0.3538.118',
+        '70.0.3538.119',
+        '70.0.3538.12',
+        '70.0.3538.120',
+        '70.0.3538.121',
+        '70.0.3538.122',
+        '70.0.3538.123',
+        '70.0.3538.124',
+        '70.0.3538.13',
+        '70.0.3538.14',
+        '70.0.3538.15',
+        '70.0.3538.16',
+        '70.0.3538.17',
+        '70.0.3538.18',
+        '70.0.3538.19',
+        '70.0.3538.2',
+        '70.0.3538.20',
+        '70.0.3538.21',
+        '70.0.3538.22',
+        '70.0.3538.23',
+        '70.0.3538.24',
+        '70.0.3538.25',
+        '70.0.3538.26',
+        '70.0.3538.27',
+        '70.0.3538.28',
+        '70.0.3538.29',
+        '70.0.3538.3',
+        '70.0.3538.30',
+        '70.0.3538.31',
+        '70.0.3538.32',
+        '70.0.3538.33',
+        '70.0.3538.34',
+        '70.0.3538.35',
+        '70.0.3538.36',
+        '70.0.3538.37',
+        '70.0.3538.38',
+        '70.0.3538.39',
+        '70.0.3538.4',
+        '70.0.3538.40',
+        '70.0.3538.41',
+        '70.0.3538.42',
+        '70.0.3538.43',
+        '70.0.3538.44',
+        '70.0.3538.45',
+        '70.0.3538.46',
+        '70.0.3538.47',
+        '70.0.3538.48',
+        '70.0.3538.49',
+        '70.0.3538.5',
+        '70.0.3538.50',
+        '70.0.3538.51',
+        '70.0.3538.52',
+        '70.0.3538.53',
+        '70.0.3538.54',
+        '70.0.3538.55',
+        '70.0.3538.56',
+        '70.0.3538.57',
+        '70.0.3538.58',
+        '70.0.3538.59',
+        '70.0.3538.6',
+        '70.0.3538.60',
+        '70.0.3538.61',
+        '70.0.3538.62',
+        '70.0.3538.63',
+        '70.0.3538.64',
+        '70.0.3538.65',
+        '70.0.3538.66',
+        '70.0.3538.67',
+        '70.0.3538.69',
+        '70.0.3538.7',
+        '70.0.3538.70',
+        '70.0.3538.71',
+        '70.0.3538.72',
+        '70.0.3538.73',
+        '70.0.3538.74',
+        '70.0.3538.75',
+        '70.0.3538.76',
+        '70.0.3538.77',
+        '70.0.3538.78',
+        '70.0.3538.79',
+        '70.0.3538.8',
+        '70.0.3538.80',
+        '70.0.3538.81',
+        '70.0.3538.82',
+        '70.0.3538.83',
+        '70.0.3538.84',
+        '70.0.3538.85',
+        '70.0.3538.86',
+        '70.0.3538.87',
+        '70.0.3538.9',
+        '70.0.3538.93',
+        '70.0.3538.94',
+        '70.0.3538.95',
+        '70.0.3538.96',
+        '70.0.3538.97',
+        '70.0.3538.98',
+        '70.0.3538.99',
+        '71.0.3539.0',
+        '71.0.3539.1',
+        '71.0.3540.0',
+        '71.0.3540.1',
+        '71.0.3541.0',
+        '71.0.3541.1',
+        '71.0.3542.0',
+        '71.0.3542.1',
+        '71.0.3542.2',
+        '71.0.3543.0',
+        '71.0.3543.1',
+        '71.0.3543.2',
+        '71.0.3543.3',
+        '71.0.3543.4',
+        '71.0.3544.0',
+        '71.0.3544.1',
+        '71.0.3544.2',
+        '71.0.3544.3',
+        '71.0.3544.4',
+        '71.0.3544.5',
+        '71.0.3545.0',
+        '71.0.3545.1',
+        '71.0.3545.2',
+        '71.0.3545.3',
+        '71.0.3545.4',
+        '71.0.3546.0',
+        '71.0.3546.1',
+        '71.0.3546.2',
+        '71.0.3547.0',
+        '71.0.3547.1',
+        '71.0.3548.0',
+        '71.0.3548.1',
+        '71.0.3549.0',
+        '71.0.3549.1',
+        '71.0.3550.0',
+        '71.0.3550.1',
+        '71.0.3550.2',
+        '71.0.3550.3',
+        '71.0.3551.0',
+        '71.0.3551.1',
+        '71.0.3551.2',
+        '71.0.3551.3',
+        '71.0.3552.0',
+        '71.0.3552.1',
+        '71.0.3552.2',
+        '71.0.3552.3',
+        '71.0.3552.4',
+        '71.0.3552.5',
+        '71.0.3552.6',
+        '71.0.3553.0',
+        '71.0.3553.1',
+        '71.0.3553.2',
+        '71.0.3553.3',
+        '71.0.3554.0',
+        '71.0.3554.1',
+        '71.0.3554.2',
+        '71.0.3554.3',
+        '71.0.3554.4',
+        '71.0.3555.0',
+        '71.0.3555.1',
+        '71.0.3555.2',
+        '71.0.3556.0',
+        '71.0.3556.1',
+        '71.0.3557.0',
+        '71.0.3557.1',
+        '71.0.3557.2',
+        '71.0.3558.0',
+        '71.0.3558.1',
+        '71.0.3558.2',
+        '71.0.3559.0',
+        '71.0.3559.1',
+        '71.0.3559.2',
+        '71.0.3559.3',
+        '71.0.3559.4',
+        '71.0.3559.5',
+        '71.0.3559.6',
+        '71.0.3560.0',
+        '71.0.3560.1',
+        '71.0.3561.0',
+        '71.0.3561.1',
+        '71.0.3562.0',
+        '71.0.3562.1',
+        '71.0.3562.2',
+        '71.0.3563.0',
+        '71.0.3564.0',
+        '71.0.3564.1',
+        '71.0.3565.0',
+        '71.0.3565.1',
+        '71.0.3566.0',
+        '71.0.3566.1',
+        '71.0.3567.0',
+        '71.0.3567.1',
+        '71.0.3568.0',
+        '71.0.3568.1',
+        '71.0.3568.2',
+        '71.0.3569.0',
+        '71.0.3569.1',
+        '71.0.3570.0',
+        '71.0.3570.1',
+        '71.0.3571.0',
+        '71.0.3571.1',
+        '71.0.3571.2',
+        '71.0.3572.0',
+        '71.0.3572.1',
+        '71.0.3573.0',
+        '71.0.3573.1',
+        '71.0.3574.0',
+        '71.0.3574.1',
+        '71.0.3575.0',
+        '71.0.3575.1',
+        '71.0.3575.2',
+        '71.0.3576.0',
+        '71.0.3576.1',
+        '71.0.3576.2',
+        '71.0.3577.0',
+        '71.0.3577.1',
+        '71.0.3578.0',
+        '71.0.3578.1',
+        '71.0.3578.10',
+        '71.0.3578.100',
+        '71.0.3578.101',
+        '71.0.3578.102',
+        '71.0.3578.103',
+        '71.0.3578.104',
+        '71.0.3578.105',
+        '71.0.3578.106',
+        '71.0.3578.107',
+        '71.0.3578.108',
+        '71.0.3578.109',
+        '71.0.3578.11',
+        '71.0.3578.110',
+        '71.0.3578.111',
+        '71.0.3578.112',
+        '71.0.3578.113',
+        '71.0.3578.114',
+        '71.0.3578.115',
+        '71.0.3578.116',
+        '71.0.3578.117',
+        '71.0.3578.118',
+        '71.0.3578.119',
+        '71.0.3578.12',
+        '71.0.3578.120',
+        '71.0.3578.121',
+        '71.0.3578.122',
+        '71.0.3578.123',
+        '71.0.3578.124',
+        '71.0.3578.125',
+        '71.0.3578.126',
+        '71.0.3578.127',
+        '71.0.3578.128',
+        '71.0.3578.129',
+        '71.0.3578.13',
+        '71.0.3578.130',
+        '71.0.3578.131',
+        '71.0.3578.132',
+        '71.0.3578.133',
+        '71.0.3578.134',
+        '71.0.3578.135',
+        '71.0.3578.136',
+        '71.0.3578.137',
+        '71.0.3578.138',
+        '71.0.3578.139',
+        '71.0.3578.14',
+        '71.0.3578.140',
+        '71.0.3578.141',
+        '71.0.3578.15',
+        '71.0.3578.16',
+        '71.0.3578.17',
+        '71.0.3578.18',
+        '71.0.3578.19',
+        '71.0.3578.2',
+        '71.0.3578.20',
+        '71.0.3578.21',
+        '71.0.3578.22',
+        '71.0.3578.23',
+        '71.0.3578.24',
+        '71.0.3578.25',
+        '71.0.3578.26',
+        '71.0.3578.27',
+        '71.0.3578.28',
+        '71.0.3578.29',
+        '71.0.3578.3',
+        '71.0.3578.30',
+        '71.0.3578.31',
+        '71.0.3578.32',
+        '71.0.3578.33',
+        '71.0.3578.34',
+        '71.0.3578.35',
+        '71.0.3578.36',
+        '71.0.3578.37',
+        '71.0.3578.38',
+        '71.0.3578.39',
+        '71.0.3578.4',
+        '71.0.3578.40',
+        '71.0.3578.41',
+        '71.0.3578.42',
+        '71.0.3578.43',
+        '71.0.3578.44',
+        '71.0.3578.45',
+        '71.0.3578.46',
+        '71.0.3578.47',
+        '71.0.3578.48',
+        '71.0.3578.49',
+        '71.0.3578.5',
+        '71.0.3578.50',
+        '71.0.3578.51',
+        '71.0.3578.52',
+        '71.0.3578.53',
+        '71.0.3578.54',
+        '71.0.3578.55',
+        '71.0.3578.56',
+        '71.0.3578.57',
+        '71.0.3578.58',
+        '71.0.3578.59',
+        '71.0.3578.6',
+        '71.0.3578.60',
+        '71.0.3578.61',
+        '71.0.3578.62',
+        '71.0.3578.63',
+        '71.0.3578.64',
+        '71.0.3578.65',
+        '71.0.3578.66',
+        '71.0.3578.67',
+        '71.0.3578.68',
+        '71.0.3578.69',
+        '71.0.3578.7',
+        '71.0.3578.70',
+        '71.0.3578.71',
+        '71.0.3578.72',
+        '71.0.3578.73',
+        '71.0.3578.74',
+        '71.0.3578.75',
+        '71.0.3578.76',
+        '71.0.3578.77',
+        '71.0.3578.78',
+        '71.0.3578.79',
+        '71.0.3578.8',
+        '71.0.3578.80',
+        '71.0.3578.81',
+        '71.0.3578.82',
+        '71.0.3578.83',
+        '71.0.3578.84',
+        '71.0.3578.85',
+        '71.0.3578.86',
+        '71.0.3578.87',
+        '71.0.3578.88',
+        '71.0.3578.89',
+        '71.0.3578.9',
+        '71.0.3578.90',
+        '71.0.3578.91',
+        '71.0.3578.92',
+        '71.0.3578.93',
+        '71.0.3578.94',
+        '71.0.3578.95',
+        '71.0.3578.96',
+        '71.0.3578.97',
+        '71.0.3578.98',
+        '71.0.3578.99',
+        '72.0.3579.0',
+        '72.0.3579.1',
+        '72.0.3580.0',
+        '72.0.3580.1',
+        '72.0.3581.0',
+        '72.0.3581.1',
+        '72.0.3581.2',
+        '72.0.3581.3',
+        '72.0.3581.4',
+        '72.0.3582.0',
+        '72.0.3583.0',
+        '72.0.3583.1',
+        '72.0.3583.2',
+        '72.0.3584.0',
+        '72.0.3584.1',
+        '72.0.3585.0',
+        '72.0.3585.1',
+        '72.0.3586.0',
+        '72.0.3586.1',
+        '72.0.3586.2',
+        '72.0.3587.0',
+        '72.0.3588.0',
+        '72.0.3588.1',
+        '72.0.3589.0',
+        '72.0.3589.1',
+        '72.0.3589.2',
+        '72.0.3589.3',
+        '72.0.3590.0',
+        '72.0.3590.1',
+        '72.0.3591.0',
+        '72.0.3591.1',
+        '72.0.3591.2',
+        '72.0.3591.3',
+        '72.0.3592.0',
+        '72.0.3592.1',
+        '72.0.3592.2',
+        '72.0.3593.0',
+        '72.0.3593.1',
+        '72.0.3593.2',
+        '72.0.3594.0',
+        '72.0.3594.1',
+        '72.0.3595.0',
+        '72.0.3595.1',
+        '72.0.3595.2',
+        '72.0.3596.0',
+        '72.0.3596.1',
+        '72.0.3596.2',
+        '72.0.3597.0',
+        '72.0.3597.1',
+        '72.0.3598.0',
+        '72.0.3598.1',
+        '72.0.3599.0',
+        '72.0.3599.1',
+        '72.0.3599.2',
+        '72.0.3599.3',
+        '72.0.3600.0',
+        '72.0.3600.1',
+        '72.0.3601.0',
+        '72.0.3601.1',
+        '72.0.3602.0',
+        '72.0.3602.1',
+        '72.0.3602.2',
+        '72.0.3602.3',
+        '72.0.3603.0',
+        '72.0.3603.1',
+        '72.0.3603.2',
+        '72.0.3604.0',
+        '72.0.3604.1',
+        '72.0.3605.0',
+        '72.0.3605.1',
+        '72.0.3605.2',
+        '72.0.3605.3',
+        '72.0.3606.0',
+        '72.0.3606.1',
+        '72.0.3606.2',
+        '72.0.3607.0',
+        '72.0.3607.1',
+        '72.0.3608.0',
+        '72.0.3608.1',
+        '72.0.3608.2',
+        '72.0.3608.3',
+        '72.0.3608.4',
+        '72.0.3608.5',
+        '72.0.3609.0',
+        '72.0.3609.1',
+        '72.0.3609.2',
+        '72.0.3609.3',
+        '72.0.3610.0',
+        '72.0.3610.1',
+        '72.0.3610.2',
+        '72.0.3610.3',
+        '72.0.3610.4',
+        '72.0.3611.0',
+        '72.0.3611.1',
+        '72.0.3611.2',
+        '72.0.3612.0',
+        '72.0.3612.1',
+        '72.0.3612.2',
+        '72.0.3613.0',
+        '72.0.3613.1',
+        '72.0.3614.0',
+        '72.0.3614.1',
+        '72.0.3615.0',
+        '72.0.3615.1',
+        '72.0.3616.0',
+        '72.0.3616.1',
+        '72.0.3617.0',
+        '72.0.3617.1',
+        '72.0.3618.0',
+        '72.0.3618.1',
+        '72.0.3619.0',
+        '72.0.3619.1',
+        '72.0.3620.0',
+        '72.0.3620.1',
+        '72.0.3621.0',
+        '72.0.3621.1',
+        '72.0.3622.0',
+        '72.0.3622.1',
+        '72.0.3622.2',
+        '72.0.3622.3',
+        '72.0.3623.0',
+        '72.0.3623.1',
+        '72.0.3623.2',
+        '72.0.3623.3',
+        '72.0.3623.4',
+        '72.0.3624.0',
+        '72.0.3624.1',
+        '72.0.3624.2',
+        '72.0.3624.3',
+        '72.0.3624.4',
+        '72.0.3625.0',
+        '72.0.3625.1',
+        '72.0.3625.2',
+        '72.0.3626.0',
+        '72.0.3626.1',
+        '72.0.3626.10',
+        '72.0.3626.100',
+        '72.0.3626.101',
+        '72.0.3626.102',
+        '72.0.3626.103',
+        '72.0.3626.105',
+        '72.0.3626.107',
+        '72.0.3626.108',
+        '72.0.3626.109',
+        '72.0.3626.11',
+        '72.0.3626.110',
+        '72.0.3626.111',
+        '72.0.3626.112',
+        '72.0.3626.113',
+        '72.0.3626.114',
+        '72.0.3626.115',
+        '72.0.3626.116',
+        '72.0.3626.117',
+        '72.0.3626.118',
+        '72.0.3626.119',
+        '72.0.3626.12',
+        '72.0.3626.120',
+        '72.0.3626.121',
+        '72.0.3626.122',
+        '72.0.3626.13',
+        '72.0.3626.14',
+        '72.0.3626.15',
+        '72.0.3626.16',
+        '72.0.3626.17',
+        '72.0.3626.18',
+        '72.0.3626.19',
+        '72.0.3626.2',
+        '72.0.3626.20',
+        '72.0.3626.21',
+        '72.0.3626.22',
+        '72.0.3626.23',
+        '72.0.3626.24',
+        '72.0.3626.25',
+        '72.0.3626.26',
+        '72.0.3626.27',
+        '72.0.3626.28',
+        '72.0.3626.29',
+        '72.0.3626.3',
+        '72.0.3626.30',
+        '72.0.3626.31',
+        '72.0.3626.32',
+        '72.0.3626.33',
+        '72.0.3626.34',
+        '72.0.3626.35',
+        '72.0.3626.36',
+        '72.0.3626.37',
+        '72.0.3626.38',
+        '72.0.3626.39',
+        '72.0.3626.4',
+        '72.0.3626.40',
+        '72.0.3626.41',
+        '72.0.3626.42',
+        '72.0.3626.43',
+        '72.0.3626.44',
+        '72.0.3626.45',
+        '72.0.3626.46',
+        '72.0.3626.47',
+        '72.0.3626.48',
+        '72.0.3626.49',
+        '72.0.3626.5',
+        '72.0.3626.50',
+        '72.0.3626.51',
+        '72.0.3626.52',
+        '72.0.3626.53',
+        '72.0.3626.54',
+        '72.0.3626.55',
+        '72.0.3626.56',
+        '72.0.3626.57',
+        '72.0.3626.58',
+        '72.0.3626.59',
+        '72.0.3626.6',
+        '72.0.3626.60',
+        '72.0.3626.61',
+        '72.0.3626.62',
+        '72.0.3626.63',
+        '72.0.3626.64',
+        '72.0.3626.65',
+        '72.0.3626.66',
+        '72.0.3626.67',
+        '72.0.3626.68',
+        '72.0.3626.69',
+        '72.0.3626.7',
+        '72.0.3626.70',
+        '72.0.3626.71',
+        '72.0.3626.72',
+        '72.0.3626.73',
+        '72.0.3626.74',
+        '72.0.3626.75',
+        '72.0.3626.76',
+        '72.0.3626.77',
+        '72.0.3626.78',
+        '72.0.3626.79',
+        '72.0.3626.8',
+        '72.0.3626.80',
+        '72.0.3626.81',
+        '72.0.3626.82',
+        '72.0.3626.83',
+        '72.0.3626.84',
+        '72.0.3626.85',
+        '72.0.3626.86',
+        '72.0.3626.87',
+        '72.0.3626.88',
+        '72.0.3626.89',
+        '72.0.3626.9',
+        '72.0.3626.90',
+        '72.0.3626.91',
+        '72.0.3626.92',
+        '72.0.3626.93',
+        '72.0.3626.94',
+        '72.0.3626.95',
+        '72.0.3626.96',
+        '72.0.3626.97',
+        '72.0.3626.98',
+        '72.0.3626.99',
+        '73.0.3627.0',
+        '73.0.3627.1',
+        '73.0.3628.0',
+        '73.0.3628.1',
+        '73.0.3628.2',
+        '73.0.3628.3',
+        '73.0.3629.0',
+        '73.0.3629.1',
+        '73.0.3630.0',
+        '73.0.3630.1',
+        '73.0.3631.0',
+        '73.0.3631.1',
+        '73.0.3631.2',
+        '73.0.3632.0',
+        '73.0.3632.1',
+        '73.0.3632.2',
+        '73.0.3632.3',
+        '73.0.3632.4',
+        '73.0.3632.5',
+        '73.0.3633.0',
+        '73.0.3633.1',
+        '73.0.3633.2',
+        '73.0.3634.0',
+        '73.0.3634.1',
+        '73.0.3634.2',
+        '73.0.3635.0',
+        '73.0.3635.1',
+        '73.0.3635.2',
+        '73.0.3635.3',
+        '73.0.3636.0',
+        '73.0.3636.1',
+        '73.0.3636.2',
+        '73.0.3637.0',
+        '73.0.3637.1',
+        '73.0.3638.0',
+        '73.0.3638.1',
+        '73.0.3638.2',
+        '73.0.3639.0',
+        '73.0.3639.1',
+        '73.0.3640.0',
+        '73.0.3640.1',
+        '73.0.3641.0',
+        '73.0.3641.1',
+        '73.0.3642.0',
+        '73.0.3642.1',
+        '73.0.3643.0',
+        '73.0.3643.1',
+        '73.0.3643.2',
+        '73.0.3644.0',
+        '73.0.3645.0',
+        '73.0.3645.1',
+        '73.0.3645.2',
+        '73.0.3646.0',
+        '73.0.3646.1',
+        '73.0.3646.2',
+        '73.0.3647.0',
+        '73.0.3647.1',
+        '73.0.3647.2',
+        '73.0.3648.0',
+        '73.0.3648.1',
+        '73.0.3648.2',
+        '73.0.3649.0',
+        '73.0.3649.1',
+        '73.0.3650.0',
+        '73.0.3650.1',
+        '73.0.3651.0',
+        '73.0.3651.1',
+        '73.0.3652.0',
+        '73.0.3652.1',
+        '73.0.3653.0',
+        '73.0.3653.1',
+        '73.0.3654.0',
+        '73.0.3654.1',
+        '73.0.3655.0',
+        '73.0.3655.1',
+        '73.0.3656.0',
+        '73.0.3656.1',
+        '73.0.3657.0',
+        '73.0.3657.1',
+        '73.0.3658.0',
+        '73.0.3658.1',
+        '73.0.3659.0',
+        '73.0.3659.1',
+        '73.0.3660.0',
+        '73.0.3660.1',
+        '73.0.3660.2',
+        '73.0.3661.0',
+        '73.0.3661.1',
+        '73.0.3662.0',
+        '73.0.3662.1',
+        '73.0.3663.0',
+        '73.0.3663.1',
+        '73.0.3663.2',
+        '73.0.3664.0',
+        '73.0.3664.1',
+        '73.0.3664.2',
+        '73.0.3664.3',
+        '73.0.3664.4',
+        '73.0.3665.0',
+        '73.0.3665.1',
+        '73.0.3665.2',
+        '73.0.3665.3',
+        '73.0.3665.4',
+        '73.0.3666.0',
+        '73.0.3666.1',
+        '73.0.3667.0',
+        '73.0.3667.1',
+        '73.0.3667.2',
+        '73.0.3668.0',
+        '73.0.3668.1',
+        '73.0.3668.2',
+        '73.0.3669.0',
+        '73.0.3669.1',
+        '73.0.3670.0',
+        '73.0.3670.1',
+        '73.0.3671.0',
+        '73.0.3671.1',
+        '73.0.3671.2',
+        '73.0.3671.3',
+        '73.0.3672.0',
+        '73.0.3672.1',
+        '73.0.3673.0',
+        '73.0.3673.1',
+        '73.0.3673.2',
+        '73.0.3674.0',
+        '73.0.3674.1',
+        '73.0.3674.2',
+        '73.0.3676.0',
+        '73.0.3676.1',
+        '73.0.3677.0',
+        '73.0.3677.1',
+        '73.0.3678.0',
+        '73.0.3678.1',
+        '73.0.3678.2',
+        '73.0.3679.0',
+        '73.0.3679.1',
+        '73.0.3680.0',
+        '73.0.3680.1',
+        '73.0.3681.0',
+        '73.0.3681.1',
+        '73.0.3681.2',
+        '73.0.3681.3',
+        '73.0.3681.4',
+        '73.0.3682.0',
+        '73.0.3682.1',
+        '73.0.3683.0',
+        '73.0.3683.1',
+        '73.0.3683.10',
+        '73.0.3683.100',
+        '73.0.3683.101',
+        '73.0.3683.102',
+        '73.0.3683.103',
+        '73.0.3683.104',
+        '73.0.3683.105',
+        '73.0.3683.106',
+        '73.0.3683.107',
+        '73.0.3683.108',
+        '73.0.3683.109',
+        '73.0.3683.11',
+        '73.0.3683.110',
+        '73.0.3683.111',
+        '73.0.3683.112',
+        '73.0.3683.113',
+        '73.0.3683.114',
+        '73.0.3683.115',
+        '73.0.3683.116',
+        '73.0.3683.117',
+        '73.0.3683.118',
+        '73.0.3683.119',
+        '73.0.3683.12',
+        '73.0.3683.120',
+        '73.0.3683.121',
+        '73.0.3683.13',
+        '73.0.3683.14',
+        '73.0.3683.15',
+        '73.0.3683.16',
+        '73.0.3683.17',
+        '73.0.3683.18',
+        '73.0.3683.19',
+        '73.0.3683.2',
+        '73.0.3683.20',
+        '73.0.3683.21',
+        '73.0.3683.22',
+        '73.0.3683.23',
+        '73.0.3683.24',
+        '73.0.3683.25',
+        '73.0.3683.26',
+        '73.0.3683.27',
+        '73.0.3683.28',
+        '73.0.3683.29',
+        '73.0.3683.3',
+        '73.0.3683.31',
+        '73.0.3683.32',
+        '73.0.3683.33',
+        '73.0.3683.34',
+        '73.0.3683.35',
+        '73.0.3683.36',
+        '73.0.3683.37',
+        '73.0.3683.38',
+        '73.0.3683.39',
+        '73.0.3683.4',
+        '73.0.3683.40',
+        '73.0.3683.41',
+        '73.0.3683.42',
+        '73.0.3683.43',
+        '73.0.3683.44',
+        '73.0.3683.45',
+        '73.0.3683.46',
+        '73.0.3683.47',
+        '73.0.3683.48',
+        '73.0.3683.49',
+        '73.0.3683.5',
+        '73.0.3683.50',
+        '73.0.3683.51',
+        '73.0.3683.52',
+        '73.0.3683.53',
+        '73.0.3683.54',
+        '73.0.3683.55',
+        '73.0.3683.56',
+        '73.0.3683.57',
+        '73.0.3683.58',
+        '73.0.3683.59',
+        '73.0.3683.6',
+        '73.0.3683.60',
+        '73.0.3683.61',
+        '73.0.3683.62',
+        '73.0.3683.63',
+        '73.0.3683.64',
+        '73.0.3683.65',
+        '73.0.3683.66',
+        '73.0.3683.67',
+        '73.0.3683.68',
+        '73.0.3683.69',
+        '73.0.3683.7',
+        '73.0.3683.70',
+        '73.0.3683.71',
+        '73.0.3683.72',
+        '73.0.3683.73',
+        '73.0.3683.74',
+        '73.0.3683.75',
+        '73.0.3683.76',
+        '73.0.3683.77',
+        '73.0.3683.78',
+        '73.0.3683.79',
+        '73.0.3683.8',
+        '73.0.3683.80',
+        '73.0.3683.81',
+        '73.0.3683.82',
+        '73.0.3683.83',
+        '73.0.3683.84',
+        '73.0.3683.85',
+        '73.0.3683.86',
+        '73.0.3683.87',
+        '73.0.3683.88',
+        '73.0.3683.89',
+        '73.0.3683.9',
+        '73.0.3683.90',
+        '73.0.3683.91',
+        '73.0.3683.92',
+        '73.0.3683.93',
+        '73.0.3683.94',
+        '73.0.3683.95',
+        '73.0.3683.96',
+        '73.0.3683.97',
+        '73.0.3683.98',
+        '73.0.3683.99',
+        '74.0.3684.0',
+        '74.0.3684.1',
+        '74.0.3685.0',
+        '74.0.3685.1',
+        '74.0.3686.0',
+        '74.0.3686.1',
+        '74.0.3686.2',
+        '74.0.3686.3',
+        '74.0.3686.4',
+        '74.0.3687.0',
+        '74.0.3687.1',
+        '74.0.3687.2',
+        '74.0.3687.3',
+        '74.0.3688.0',
+        '74.0.3688.1',
+        '74.0.3689.0',
+        '74.0.3689.1',
+        '74.0.3690.0',
+        '74.0.3690.1',
+        '74.0.3691.0',
+        '74.0.3691.1',
+        '74.0.3692.0',
+        '74.0.3692.1',
+        '74.0.3693.0',
+        '74.0.3693.1',
+        '74.0.3693.2',
+        '74.0.3693.3',
+        '74.0.3693.4',
+        '74.0.3693.5',
+        '74.0.3693.6',
+        '74.0.3694.0',
+        '74.0.3694.1',
+        '74.0.3694.2',
+        '74.0.3694.3',
+        '74.0.3694.4',
+        '74.0.3694.5',
+        '74.0.3694.6',
+        '74.0.3694.7',
+        '74.0.3694.8',
+        '74.0.3696.0',
+        '74.0.3696.1',
+        '74.0.3696.2',
+        '74.0.3698.0',
+        '74.0.3699.0',
+        '74.0.3699.1',
+        '74.0.3699.2',
+        '74.0.3699.3',
+        '74.0.3700.0',
+        '74.0.3700.1',
+        '74.0.3701.0',
+        '74.0.3701.1',
+        '74.0.3702.0',
+        '74.0.3702.1',
+        '74.0.3702.2',
+        '74.0.3702.3',
+        '74.0.3703.0',
+        '74.0.3703.1',
+        '74.0.3703.2',
+        '74.0.3703.3',
+        '74.0.3704.0',
+        '74.0.3704.1',
+        '74.0.3704.2',
+        '74.0.3704.3',
+        '74.0.3704.4',
+        '74.0.3704.5',
+        '74.0.3704.6',
+        '74.0.3704.7',
+        '74.0.3704.8',
+        '74.0.3704.9',
+        '74.0.3705.0',
+        '74.0.3705.1',
+        '74.0.3706.0',
+        '74.0.3706.1',
+        '74.0.3706.2',
+        '74.0.3706.3',
+        '74.0.3706.4',
+        '74.0.3706.5',
+        '74.0.3706.6',
+        '74.0.3706.7',
+        '74.0.3708.0',
+        '74.0.3709.0',
+        '74.0.3709.1',
+        '74.0.3710.0',
+        '74.0.3710.1',
+        '74.0.3710.2',
+        '74.0.3711.0',
+        '74.0.3711.1',
+        '74.0.3711.2',
+        '74.0.3712.0',
+        '74.0.3712.1',
+        '74.0.3712.2',
+        '74.0.3713.0',
+        '74.0.3713.1',
+        '74.0.3713.3',
+        '74.0.3714.0',
+        '74.0.3714.1',
+        '74.0.3714.2',
+        '74.0.3715.0',
+        '74.0.3715.1',
+        '74.0.3716.0',
+        '74.0.3716.1',
+        '74.0.3717.0',
+        '74.0.3717.1',
+        '74.0.3717.2',
+        '74.0.3718.0',
+        '74.0.3718.1',
+        '74.0.3718.2',
+        '74.0.3718.3',
+        '74.0.3718.4',
+        '74.0.3718.5',
+        '74.0.3718.6',
+        '74.0.3718.7',
+        '74.0.3718.8',
+        '74.0.3718.9',
+        '74.0.3719.0',
+        '74.0.3719.1',
+        '74.0.3719.2',
+        '74.0.3719.3',
+        '74.0.3719.4',
+        '74.0.3719.5',
+        '74.0.3720.0',
+        '74.0.3720.1',
+        '74.0.3720.2',
+        '74.0.3720.3',
+        '74.0.3720.4',
+        '74.0.3720.5',
+        '74.0.3720.6',
+        '74.0.3721.0',
+        '74.0.3721.1',
+        '74.0.3721.2',
+        '74.0.3721.3',
+        '74.0.3722.0',
+        '74.0.3722.1',
+        '74.0.3723.0',
+        '74.0.3723.1',
+        '74.0.3724.0',
+        '74.0.3724.1',
+        '74.0.3724.2',
+        '74.0.3724.3',
+        '74.0.3724.4',
+        '74.0.3724.5',
+        '74.0.3724.6',
+        '74.0.3724.7',
+        '74.0.3724.8',
+        '74.0.3725.0',
+        '74.0.3725.1',
+        '74.0.3725.2',
+        '74.0.3725.3',
+        '74.0.3725.4',
+        '74.0.3726.0',
+        '74.0.3726.1',
+        '74.0.3726.2',
+        '74.0.3726.3',
+        '74.0.3726.4',
+        '74.0.3728.0',
+        '74.0.3729.0',
+        '74.0.3729.1',
+        '74.0.3729.10',
+        '74.0.3729.100',
+        '74.0.3729.101',
+        '74.0.3729.102',
+        '74.0.3729.103',
+        '74.0.3729.104',
+        '74.0.3729.105',
+        '74.0.3729.106',
+        '74.0.3729.107',
+        '74.0.3729.108',
+        '74.0.3729.109',
+        '74.0.3729.11',
+        '74.0.3729.110',
+        '74.0.3729.111',
+        '74.0.3729.112',
+        '74.0.3729.113',
+        '74.0.3729.114',
+        '74.0.3729.115',
+        '74.0.3729.116',
+        '74.0.3729.117',
+        '74.0.3729.118',
+        '74.0.3729.119',
+        '74.0.3729.12',
+        '74.0.3729.120',
+        '74.0.3729.121',
+        '74.0.3729.122',
+        '74.0.3729.123',
+        '74.0.3729.124',
+        '74.0.3729.125',
+        '74.0.3729.126',
+        '74.0.3729.127',
+        '74.0.3729.128',
+        '74.0.3729.129',
+        '74.0.3729.13',
+        '74.0.3729.14',
+        '74.0.3729.15',
+        '74.0.3729.16',
+        '74.0.3729.17',
+        '74.0.3729.18',
+        '74.0.3729.19',
+        '74.0.3729.2',
+        '74.0.3729.20',
+        '74.0.3729.21',
+        '74.0.3729.22',
+        '74.0.3729.23',
+        '74.0.3729.24',
+        '74.0.3729.25',
+        '74.0.3729.26',
+        '74.0.3729.27',
+        '74.0.3729.28',
+        '74.0.3729.29',
+        '74.0.3729.3',
+        '74.0.3729.30',
+        '74.0.3729.31',
+        '74.0.3729.32',
+        '74.0.3729.33',
+        '74.0.3729.34',
+        '74.0.3729.35',
+        '74.0.3729.36',
+        '74.0.3729.37',
+        '74.0.3729.38',
+        '74.0.3729.39',
+        '74.0.3729.4',
+        '74.0.3729.40',
+        '74.0.3729.41',
+        '74.0.3729.42',
+        '74.0.3729.43',
+        '74.0.3729.44',
+        '74.0.3729.45',
+        '74.0.3729.46',
+        '74.0.3729.47',
+        '74.0.3729.48',
+        '74.0.3729.49',
+        '74.0.3729.5',
+        '74.0.3729.50',
+        '74.0.3729.51',
+        '74.0.3729.52',
+        '74.0.3729.53',
+        '74.0.3729.54',
+        '74.0.3729.55',
+        '74.0.3729.56',
+        '74.0.3729.57',
+        '74.0.3729.58',
+        '74.0.3729.59',
+        '74.0.3729.6',
+        '74.0.3729.60',
+        '74.0.3729.61',
+        '74.0.3729.62',
+        '74.0.3729.63',
+        '74.0.3729.64',
+        '74.0.3729.65',
+        '74.0.3729.66',
+        '74.0.3729.67',
+        '74.0.3729.68',
+        '74.0.3729.69',
+        '74.0.3729.7',
+        '74.0.3729.70',
+        '74.0.3729.71',
+        '74.0.3729.72',
+        '74.0.3729.73',
+        '74.0.3729.74',
+        '74.0.3729.75',
+        '74.0.3729.76',
+        '74.0.3729.77',
+        '74.0.3729.78',
+        '74.0.3729.79',
+        '74.0.3729.8',
+        '74.0.3729.80',
+        '74.0.3729.81',
+        '74.0.3729.82',
+        '74.0.3729.83',
+        '74.0.3729.84',
+        '74.0.3729.85',
+        '74.0.3729.86',
+        '74.0.3729.87',
+        '74.0.3729.88',
+        '74.0.3729.89',
+        '74.0.3729.9',
+        '74.0.3729.90',
+        '74.0.3729.91',
+        '74.0.3729.92',
+        '74.0.3729.93',
+        '74.0.3729.94',
+        '74.0.3729.95',
+        '74.0.3729.96',
+        '74.0.3729.97',
+        '74.0.3729.98',
+        '74.0.3729.99',
+        '75.0.3730.0',
+        '75.0.3730.1',
+        '75.0.3730.2',
+        '75.0.3730.3',
+        '75.0.3730.4',
+        '75.0.3730.5',
+        '75.0.3731.0',
+        '75.0.3731.2',
+        '75.0.3731.3',
+        '75.0.3732.0',
+        '75.0.3732.1',
+        '75.0.3733.0',
+        '75.0.3733.1',
+        '75.0.3734.0',
+        '75.0.3734.1',
+        '75.0.3735.0',
+        '75.0.3735.1',
+        '75.0.3736.0',
+        '75.0.3736.1',
+        '75.0.3737.0',
+        '75.0.3737.1',
+        '75.0.3738.0',
+        '75.0.3738.1',
+        '75.0.3738.2',
+        '75.0.3738.3',
+        '75.0.3738.4',
+        '75.0.3739.0',
+        '75.0.3739.1',
+        '75.0.3740.0',
+        '75.0.3740.1',
+        '75.0.3740.2',
+        '75.0.3740.3',
+        '75.0.3740.4',
+        '75.0.3740.5',
+        '75.0.3741.0',
+        '75.0.3741.1',
+        '75.0.3741.2',
+        '75.0.3744.0',
+        '75.0.3744.1',
+        '75.0.3744.2',
+        '75.0.3745.0',
+        '75.0.3745.1',
+        '75.0.3745.2',
+        '75.0.3745.3',
+        '75.0.3745.4',
+        '75.0.3745.5',
+        '75.0.3746.0',
+        '75.0.3746.1',
+        '75.0.3746.2',
+        '75.0.3746.3',
+        '75.0.3746.4',
+        '75.0.3747.0',
+        '75.0.3747.1',
+        '75.0.3748.0',
+        '75.0.3748.1',
+        '75.0.3749.0',
+        '75.0.3749.1',
+        '75.0.3749.2',
+        '75.0.3749.3',
+        '75.0.3750.0',
+        '75.0.3751.0',
+        '75.0.3751.1',
+        '75.0.3752.0',
+        '75.0.3752.1',
+        '75.0.3752.2',
+        '75.0.3753.0',
+        '75.0.3753.1',
+        '75.0.3753.2',
+        '75.0.3753.3',
+        '75.0.3753.4',
+        '75.0.3754.0',
+        '75.0.3754.1',
+        '75.0.3754.2',
+        '75.0.3755.0',
+        '75.0.3755.1',
+        '75.0.3755.2',
+        '75.0.3755.3',
+        '75.0.3756.0',
+        '75.0.3756.1',
+        '75.0.3757.0',
+        '75.0.3757.1',
+        '75.0.3758.0',
+        '75.0.3758.1',
+        '75.0.3759.0',
+        '75.0.3759.1',
+        '75.0.3759.2',
+        '75.0.3759.3',
+        '75.0.3759.4',
+        '75.0.3759.5',
+        '75.0.3759.6',
+        '75.0.3759.7',
+        '75.0.3759.8',
+        '75.0.3760.0',
+        '75.0.3761.0',
+        '75.0.3761.1',
+        '75.0.3761.2',
+        '75.0.3761.3',
+        '75.0.3761.4',
+        '75.0.3762.0',
+        '75.0.3762.1',
+        '75.0.3763.0',
+        '75.0.3763.1',
+        '75.0.3763.2',
+        '75.0.3764.0',
+        '75.0.3764.1',
+        '75.0.3765.0',
+        '75.0.3765.1',
+        '75.0.3766.0',
+        '75.0.3766.1',
+        '75.0.3766.2',
+        '75.0.3766.3',
+        '75.0.3767.0',
+        '75.0.3767.1',
+        '75.0.3767.2',
+        '75.0.3768.0',
+        '75.0.3768.1',
+        '75.0.3768.2',
+        '75.0.3768.3',
+        '75.0.3768.4',
+        '75.0.3768.5',
+        '75.0.3768.6',
+        '75.0.3769.0',
+        '75.0.3769.1',
+        '75.0.3769.2',
+        '75.0.3769.3',
+        '75.0.3769.4',
+        '75.0.3769.5',
+        '75.0.3770.0',
+        '75.0.3770.1',
+        '75.0.3770.10',
+        '75.0.3770.11',
+        '75.0.3770.12',
+        '75.0.3770.13',
+        '75.0.3770.14',
+        '75.0.3770.15',
+        '75.0.3770.2',
+        '75.0.3770.3',
+        '75.0.3770.4',
+        '75.0.3770.5',
+        '75.0.3770.6',
+        '75.0.3770.7',
+        '75.0.3770.8',
+        '75.0.3770.9',
+        '76.0.3771.0',
+        '76.0.3771.1',
+        '76.0.3772.0',
+        '76.0.3772.1',
+        '76.0.3773.0',
+        '76.0.3773.1',
+        '76.0.3774.0',
+        '76.0.3774.1',
+        '76.0.3775.0',
+        '76.0.3775.1',
+        '76.0.3775.2',
+        '76.0.3775.3',
+        '76.0.3775.4',
+        '76.0.3775.5',
+        '76.0.3776.0',
+        '76.0.3776.1',
+        '76.0.3776.2',
+        '76.0.3776.3',
+        '76.0.3776.4',
+        '76.0.3777.0',
+        '76.0.3777.1',
+        '76.0.3778.0',
+        '76.0.3778.1',
+        '76.0.3779.0',
+        '76.0.3779.1',
+        '76.0.3780.0',
+        '76.0.3780.1',
+        '76.0.3780.2',
+        '76.0.3780.3',
     )
     return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
 
 
 std_headers = {
     'User-Agent': random_user_agent(),
-    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',
     'Accept-Language': 'en-us,en;q=0.5',
 }
 
-
+# TODO: only used by CeskaTeleviseIE, replace by _USER_AGENT_TPL % _CHROME_VERSIONS[0] on UA freeze
 USER_AGENTS = {
     'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
 }

From 504543b2e04b916982dfe3d9f2bf8dd5d9124f96 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Jan 2022 20:42:42 +0000
Subject: [PATCH 101/394] Recognise FLAC audio in parse_codecs()

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f1139e61575..cd32c396813 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4310,7 +4310,7 @@ def parse_codecs(codecs_str):
         if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
             if not vcodec:
                 vcodec = full_codec
-        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
+        elif codec in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
             if not acodec:
                 acodec = full_codec
         else:

From 70b7b0f3e3f64de60d6df48f7c21f3665098569a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Jan 2022 00:04:09 +0000
Subject: [PATCH 102/394] Add parsing YYYYMMDD dates, also in Nov/Dec (yt-dlp
 PR #2094)

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 7c6a382fd9a..20183da1a36 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -346,6 +346,7 @@ def test_unified_dates(self):
         self.assertEqual(unified_strdate('Sep 2nd, 2013'), '20130902')
         self.assertEqual(unified_strdate('November 3rd, 2019'), '20191103')
         self.assertEqual(unified_strdate('October 23rd, 2005'), '20051023')
+        self.assertEqual(unified_strdate('20211221'), '20211221')
 
     def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cd32c396813..349d97f4c9e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1735,6 +1735,9 @@ def random_user_agent():
     '%Y/%m/%d',
     '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
+    '%Y%m%d%H%M',
+    '%Y%m%d%H%M%S',
+    '%Y%m%d',
     '%Y-%m-%d %H:%M',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',

From e0a1fe44eee68c0caf34476824c5406b1c4cabef Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Jan 2022 00:59:08 +0000
Subject: [PATCH 103/394] Improve parse_count() with single regex, based on
 yt-dlp 352d5da81219e2675ef8cac9383ab0dfbd161a19

---
 test/test_utils.py  |  7 +++++++
 youtube_dl/utils.py | 19 ++++++++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 20183da1a36..88d2da0c32f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1116,9 +1116,16 @@ def test_parse_count(self):
         self.assertEqual(parse_count('1000'), 1000)
         self.assertEqual(parse_count('1.000'), 1000)
         self.assertEqual(parse_count('1.1k'), 1100)
+        self.assertEqual(parse_count('1.1 k'), 1100)
+        self.assertEqual(parse_count('1,1 k'), 1100)
+        self.assertEqual(parse_count('1,1kk'), 1100000)
+        self.assertEqual(parse_count('100 views'), 100)
+        self.assertEqual(parse_count('1,100 views'), 1100)
         self.assertEqual(parse_count('1.1kk'), 1100000)
         self.assertEqual(parse_count('1.1kk '), 1100000)
         self.assertEqual(parse_count('1.1kk views'), 1100000)
+        self.assertEqual(parse_count('10M views'), 10000000)
+        self.assertEqual(parse_count('has 10M views'), 10000000)
 
     def test_parse_resolution(self):
         self.assertEqual(parse_resolution(None), {})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 349d97f4c9e..ba85a90af24 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3542,10 +3542,11 @@ def parse_count(s):
     if s is None:
         return None
 
-    s = s.strip()
-
-    if re.match(r'^[\d,.]+$', s):
-        return str_to_int(s)
+    m = re.match(r'^(?:[^\d]+\s+)?(?P<val>(?P<num>[\d,.]+)(?P<rest>[\w\s]+?)?)(?:\s|\s*$)', s)
+    if m:
+        if not m.group('rest'):
+            return str_to_int(m.group('num'))
+        s = m.group('val')
 
     _UNIT_TABLE = {
         'k': 1000,
@@ -3554,9 +3555,17 @@ def parse_count(s):
         'M': 1000 ** 2,
         'kk': 1000 ** 2,
         'KK': 1000 ** 2,
+        'b': 1000 ** 3,
+        'B': 1000 ** 3,
     }
 
-    return lookup_unit_table(_UNIT_TABLE, s)
+    ret = lookup_unit_table(_UNIT_TABLE, s)
+    if ret is not None:
+        return ret
+
+    s = m and m.group('num')
+    if s is not None:
+        return str_to_int(s)
 
 
 def parse_resolution(s):

From b8344046138083152d9b0d9a5cc8a11e28faf621 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Jan 2022 02:29:33 +0000
Subject: [PATCH 104/394] Fix/improve InAdvancePagedList, from yt-dlp
 d37707bda466282b66ce381cbfd734f899f9e06e

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ba85a90af24..a85a70b86db 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3931,7 +3931,8 @@ def getslice(self, start=0, end=None):
         res = []
         start_page = start // self._pagesize
         end_page = (
-            self._pagecount if end is None else (end // self._pagesize + 1))
+            self._pagecount if end is None
+            else min(self._pagecount, end // self._pagesize + 1))
         skip_elems = start - start_page * self._pagesize
         only_more = None if end is None else end - start
         for pagenum in range(start_page, end_page):

From 51d3d0c9f61f373c0fd6130f9fc481051d36ef99 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Jan 2022 02:51:25 +0000
Subject: [PATCH 105/394] Handle ss:xxx in parse_duration(), based on yt-dlp
 8bd1c00bf399ba7002fc21cd399c931e9d301bd8

---
 test/test_utils.py  |  2 ++
 youtube_dl/utils.py | 15 +++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 88d2da0c32f..df53a57fb74 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -617,6 +617,8 @@ def test_parse_duration(self):
         self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
         self.assertEqual(parse_duration('PT00H03M30SZ'), 210)
         self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88)
+        self.assertEqual(parse_duration('01:02:03:050'), 3723.05)
+        self.assertEqual(parse_duration('103:050'), 103.05)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a85a70b86db..cf4f7a75d96 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3754,12 +3754,14 @@ def parse_duration(s):
     if not isinstance(s, compat_basestring):
         return None
 
-    s = s.strip()
-
-    days, hours, mins, secs, ms = [None] * 5
-    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
+    m = re.match(r'''(?x)\s*
+            (?P<before_secs>
+                (?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?
+            (?P<secs>(?(before_secs)[0-9]{1,2}|[0-9]+))
+            (?P<ms>[.:][0-9]+)?Z?\s*$
+        ''', s)
     if m:
-        days, hours, mins, secs, ms = m.groups()
+        days, hours, mins, secs, ms = m.group('days', 'hours', 'mins', 'secs', 'ms')
     else:
         m = re.match(
             r'''(?ix)(?:P?
@@ -3791,6 +3793,7 @@ def parse_duration(s):
             m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
             if m:
                 hours, mins = m.groups()
+                days = secs = ms = None
             else:
                 return None
 
@@ -3804,7 +3807,7 @@ def parse_duration(s):
     if days:
         duration += float(days) * 24 * 60 * 60
     if ms:
-        duration += float(ms)
+        duration += float(ms.replace(':', '.'))
     return duration
 
 

From 5c9197cf2a82facb86891c98db88e8da1ed240d0 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 28 Jan 2022 17:18:10 +0100
Subject: [PATCH 106/394] version

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b82fbc702ff..0eaa03aeddc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.12.17'
+__version__ = '2021.01.30'

From d6f6fd47938dd1f79a3c056f9bcf2de0dd7b576b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Jan 2022 00:02:56 +0000
Subject: [PATCH 107/394] Support Youtube Shorts URL format

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63918924df8..e001ea2e62c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -416,6 +416,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
+                             |shorts/
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
@@ -1118,6 +1119,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # YT 'Shorts'
+            'url': 'https://youtube.com/shorts/4L2J27mJ3Dc',
+            'info_dict': {
+                'id': '4L2J27mJ3Dc',
+                'ext': 'mp4',
+                'upload_date': '20211025',
+                'uploader': 'Charlie Berens',
+                'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
+                'uploader_id': 'fivedlrmilkshake',
+                'title': 'Midwest Squid Game #Shorts',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},

From fe8f07fd370de14fe921ed240750e513b9558afa Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Thu, 3 Feb 2022 08:22:22 +0100
Subject: [PATCH 108/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0eaa03aeddc..2eade85593c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.30'
+__version__ = '2021.02.03'

From 0ec3ca9f803af1c2f98ae374d7074d0d518ceb86 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:24:03 +0000
Subject: [PATCH 109/394] Find TV2DK Kaltura ID in Nuxt.js page format

---
 youtube_dl/extractor/tv2dk.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 8bd5fd6401f..106a081e1e2 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -41,8 +41,16 @@ class TV2DKIE(InfoExtractor):
             'duration': 1347,
             'view_count': int,
         },
-        'params': {
-            'skip_download': True,
+        'add_ie': ['Kaltura'],
+    }, {
+        'url': 'https://www.tv2lorry.dk/gadekamp/gadekamp-6-hoejhuse-i-koebenhavn',
+        'info_dict': {
+            'id': '1_7iwll9n0',
+            'ext': 'mp4',
+            'upload_date': '20211027',
+            'title': 'Gadekamp #6 - Højhuse i København',
+            'uploader_id': 'tv2lorry',
+            'timestamp': 1635345229,
         },
         'add_ie': ['Kaltura'],
     }, {
@@ -91,7 +99,8 @@ def add_entry(partner_id, kaltura_id):
             add_entry(partner_id, kaltura_id)
         if not entries:
             kaltura_id = self._search_regex(
-                r'entry_id\s*:\s*["\']([0-9a-z_]+)', webpage, 'kaltura id')
+                (r'entry_id\s*:\s*["\']([0-9a-z_]+)',
+                 r'\\u002FentryId\\u002F(\w+)\\u002F'), webpage, 'kaltura id')
             partner_id = self._search_regex(
                 (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
                 'partner id')

From 7215bb7edb5c698f2e51fa242ad0542ceb523f4f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:38:44 +0000
Subject: [PATCH 110/394] Return the item itself if playlist has one entry

Removes playlist spam from log
---
 youtube_dl/extractor/tv2dk.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 106a081e1e2..ec5cbdf0353 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -105,6 +105,8 @@ def add_entry(partner_id, kaltura_id):
                 (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
                 'partner id')
             add_entry(partner_id, kaltura_id)
+        if len(entries) == 1:
+            return entries[0]
         return self.playlist_result(entries)
 
 

From 96ac0a9fa67f2f3a09ff83618844e60992458f63 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:29:41 +0000
Subject: [PATCH 111/394] Back-port yt-dlp Viki extractor

From https://github.com/yt-dlp/yt-dlp/pull/2540
---
 youtube_dl/extractor/viki.py | 339 +++++++++++++++--------------------
 1 file changed, 146 insertions(+), 193 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 2e9cbf1489f..2ddca0ca630 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,38 +1,29 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
 import hashlib
 import hmac
-import itertools
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_parse_urlparse,
-)
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_iso8601,
-    sanitized_Request,
-    std_headers,
     try_get,
 )
 
 
 class VikiBaseIE(InfoExtractor):
     _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
-    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
-    _API_URL_TEMPLATE = 'https://api.viki.io%s&sig=%s'
+    _API_URL_TEMPLATE = 'https://api.viki.io%s'
 
+    _DEVICE_ID = '112395910d'
     _APP = '100005a'
-    _APP_VERSION = '6.0.0'
-    _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
+    _APP_VERSION = '6.11.3'
+    _APP_SECRET = 'd96704b180208dbb2efa30fe44c48bd8690441af9f567ba8fd710a72badc85198f7472'
 
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'viki'
@@ -45,43 +36,60 @@ class VikiBaseIE(InfoExtractor):
         'paywall': 'Sorry, this content is only available to Viki Pass Plus subscribers',
     }
 
-    def _prepare_call(self, path, timestamp=None, post_data=None):
+    def _stream_headers(self, timestamp, sig):
+        return {
+            'X-Viki-manufacturer': 'vivo',
+            'X-Viki-device-model': 'vivo 1606',
+            'X-Viki-device-os-ver': '6.0.1',
+            'X-Viki-connection-type': 'WIFI',
+            'X-Viki-carrier': '',
+            'X-Viki-as-id': '100005a-1625321982-3932',
+            'timestamp': str(timestamp),
+            'signature': str(sig),
+            'x-viki-app-ver': self._APP_VERSION
+        }
+
+    def _api_query(self, path, version=4, **kwargs):
         path += '?' if '?' not in path else '&'
-        if not timestamp:
-            timestamp = int(time.time())
-        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        app = self._APP
+        query = '/v{version}/{path}app={app}'.format(**locals())
         if self._token:
             query += '&token=%s' % self._token
+        return query + ''.join('&{name}={val}.format(**locals())' for name, val in kwargs.items())
+
+    def _sign_query(self, path):
+        timestamp = int(time.time())
+        query = self._api_query(path, version=5)
         sig = hmac.new(
             self._APP_SECRET.encode('ascii'),
-            query.encode('ascii'),
-            hashlib.sha1
-        ).hexdigest()
-        url = self._API_URL_TEMPLATE % (query, sig)
-        return sanitized_Request(
-            url, json.dumps(post_data).encode('utf-8')) if post_data else url
-
-    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
+            '{query}&t={timestamp}'.format(**locals()).encode('ascii'),
+            hashlib.sha1).hexdigest()
+        return timestamp, sig, self._API_URL_TEMPLATE % query
+
+    def _call_api(
+            self, path, video_id, note='Downloading JSON metadata', data=None, query=None, fatal=True):
+        if query is None:
+            timestamp, sig, url = self._sign_query(path)
+        else:
+            url = self._API_URL_TEMPLATE % self._api_query(path, version=4)
         resp = self._download_json(
-            self._prepare_call(path, timestamp, post_data), video_id, note,
-            headers={'x-viki-app-ver': self._APP_VERSION})
-
-        error = resp.get('error')
-        if error:
-            if error == 'invalid timestamp':
-                resp = self._download_json(
-                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
-                    video_id, '%s (retry)' % note)
-                error = resp.get('error')
-            if error:
-                self._raise_error(resp['error'])
+            url, video_id, note, fatal=fatal, query=query,
+            data=json.dumps(data).encode('utf-8') if data else None,
+            headers=({'x-viki-app-ver': self._APP_VERSION} if data
+                     else self._stream_headers(timestamp, sig) if query is None
+                     else None), expected_status=400) or {}
 
+        self._raise_error(resp.get('error'), fatal)
         return resp
 
-    def _raise_error(self, error):
-        raise ExtractorError(
-            '%s returned error: %s' % (self.IE_NAME, error),
-            expected=True)
+    def _raise_error(self, error, fatal=True):
+        if error is None:
+            return
+        msg = '%s said: %s' % (self.IE_NAME, error)
+        if fatal:
+            raise ExtractorError(msg, expected=True)
+        else:
+            self.report_warning(msg)
 
     def _check_errors(self, data):
         for reason, status in (data.get('blocking') or {}).items():
@@ -90,9 +98,10 @@ def _check_errors(self, data):
                 if reason == 'geo':
                     self.raise_geo_restricted(msg=message)
                 elif reason == 'paywall':
+                    if try_get(data, lambda x: x['paywallable']['tvod']):
+                        self._raise_error('This video is for rent only or TVOD (Transactional Video On demand)')
                     self.raise_login_required(message)
-                raise ExtractorError('%s said: %s' % (
-                    self.IE_NAME, message), expected=True)
+                self._raise_error(message)
 
     def _real_initialize(self):
         self._login()
@@ -102,35 +111,39 @@ def _login(self):
         if username is None:
             return
 
-        login_form = {
-            'login_id': username,
-            'password': password,
-        }
-
-        login = self._call_api(
-            'sessions.json', None,
-            'Logging in', post_data=login_form)
-
-        self._token = login.get('token')
+        self._token = self._call_api(
+            'sessions.json', None, 'Logging in', fatal=False,
+            data={'username': username, 'password': password}).get('token')
         if not self._token:
-            self.report_warning('Unable to get session token, login has probably failed')
+            self.report_warning('Login Failed: Unable to get session token')
 
     @staticmethod
-    def dict_selection(dict_obj, preferred_key, allow_fallback=True):
+    def dict_selection(dict_obj, preferred_key):
         if preferred_key in dict_obj:
-            return dict_obj.get(preferred_key)
-
-        if not allow_fallback:
-            return
-
-        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
-        return filtered_dict[0] if filtered_dict else None
+            return dict_obj[preferred_key]
+        return (list(filter(None, dict_obj.values())) or [None])[0]
 
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
     _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
+        'note': 'Free non-DRM video with storyboards in MPD',
+        'url': 'https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1',
+        'info_dict': {
+            'id': '1175236v',
+            'ext': 'mp4',
+            'title': 'Choosing Spouse by Lottery - Episode 1',
+            'timestamp': 1606463239,
+            'age_limit': 12,
+            'uploader': 'FCC',
+            'upload_date': '20201127',
+        },
+        'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
             'id': '1023585v',
@@ -146,7 +159,7 @@ class VikiIE(VikiBaseIE):
         'params': {
             'format': 'bestvideo',
         },
-        'skip': 'Blocked in the US',
+        'skip': 'Content is only available to Viki Pass Plus subscribers',
         'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
     }, {
         # clip
@@ -178,11 +191,11 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         },
-        'skip': 'Blocked in the US',
+        'skip': 'Page not found!',
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
-        'md5': '0a53dc252e6e690feccd756861495a8c',
+        'md5': '670440c79f7109ca6564d4c7f24e3e81',
         'info_dict': {
             'id': '44699v',
             'ext': 'mp4',
@@ -193,7 +206,7 @@ class VikiIE(VikiBaseIE):
             'upload_date': '20100405',
             'uploader': 'group8',
             'like_count': int,
-            'age_limit': 13,
+            'age_limit': 15,
             'episode_number': 1,
         },
         'params': {
@@ -224,7 +237,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # non-English description
         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
-        'md5': '41faaba0de90483fb4848952af7c7d0d',
+        'md5': '78bf49fdaa51f9e7f9150262a9ef9bdf',
         'info_dict': {
             'id': '158036v',
             'ext': 'mp4',
@@ -232,8 +245,8 @@ class VikiIE(VikiBaseIE):
             'upload_date': '20111122',
             'timestamp': 1321985454,
             'description': 'md5:44b1e46619df3a072294645c770cef36',
-            'title': 'Love In Magic',
-            'age_limit': 13,
+            'title': 'Love in Magic',
+            'age_limit': 15,
         },
         'params': {
             'format': 'bestvideo',
@@ -244,45 +257,53 @@ class VikiIE(VikiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        resp = self._download_json(
-            'https://www.viki.com/api/videos/' + video_id,
-            video_id, 'Downloading video JSON', headers={
-                'x-client-user-agent': std_headers['User-Agent'],
-                'x-viki-app-ver': '3.0.0',
-            })
-        video = resp['video']
+        video = self._call_api('videos/{0}.json'.format(video_id), video_id, 'Downloading video JSON', query={})
 
         self._check_errors(video)
 
-        title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
+        title = try_get(video, lambda x: x['titles']['en'], str)
         episode_number = int_or_none(video.get('number'))
         if not title:
             title = 'Episode %d' % episode_number if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = try_get(video, lambda x: x['container']['titles'], dict) or {}
             container_title = self.dict_selection(container_titles, 'en')
-            title = '%s - %s' % (container_title, title)
+            if container_title and title == video_id:
+                title = container_title
+            else:
+                title = '%s - %s' % (container_title, title)
+
+        resp = self._call_api(
+            'playback_streams/%s.json?drms=dt3&device_id=%s' % (video_id, self._DEVICE_ID),
+            video_id, 'Downloading video streams JSON')['main'][0]
+
+        mpd_url = resp['url']
+        # 720p is hidden in another MPD which can be found in the current manifest content
+        mpd_content = self._download_webpage(mpd_url, video_id, note='Downloading initial MPD manifest')
+        mpd_url = self._search_regex(
+            r'(?mi)<BaseURL>(http.+.mpd)', mpd_content, 'new manifest', default=mpd_url)
+        if 'mpdhd_high' not in mpd_url:
+            # Modify the URL to get 1080p
+            mpd_url = mpd_url.replace('mpdhd', 'mpdhd_high')
+        formats = self._extract_mpd_formats(mpd_url, video_id)
+        self._sort_formats(formats)
 
         description = self.dict_selection(video.get('descriptions', {}), 'en')
-
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail['url'],
+        } for thumbnail_id, thumbnail in (video.get('images') or {}).items() if thumbnail.get('url')]
         like_count = int_or_none(try_get(video, lambda x: x['likes']['count']))
 
-        thumbnails = []
-        for thumbnail_id, thumbnail in (video.get('images') or {}).items():
-            thumbnails.append({
-                'id': thumbnail_id,
-                'url': thumbnail.get('url'),
-            })
-
-        subtitles = {}
-        for subtitle_lang, _ in (video.get('subtitle_completions') or {}).items():
-            subtitles[subtitle_lang] = [{
-                'ext': subtitles_format,
-                'url': self._prepare_call(
-                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
-            } for subtitles_format in ('srt', 'vtt')]
-
-        result = {
+        stream_id = try_get(resp, lambda x: x['properties']['track']['stream_id'])
+        subtitles = dict((lang, [{
+            'ext': ext,
+            'url': self._API_URL_TEMPLATE % self._api_query(
+                'videos/{0}/auth_subtitles/{1}.{2}'.format(video_id, lang, ext), stream_id=stream_id)
+        } for ext in ('srt', 'vtt')]) for lang in (video.get('subtitle_completions') or {}).keys())
+
+        return {
             'id': video_id,
+            'formats': formats,
             'title': title,
             'description': description,
             'duration': int_or_none(video.get('duration')),
@@ -296,79 +317,6 @@ def _real_extract(self, url):
             'episode_number': episode_number,
         }
 
-        formats = []
-
-        def add_format(format_id, format_dict, protocol='http'):
-            # rtmps URLs does not seem to work
-            if protocol == 'rtmps':
-                return
-            format_url = format_dict.get('url')
-            if not format_url:
-                return
-            qs = compat_parse_qs(compat_urllib_parse_urlparse(format_url).query)
-            stream = qs.get('stream', [None])[0]
-            if stream:
-                format_url = base64.b64decode(stream).decode()
-            if format_id in ('m3u8', 'hls'):
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4',
-                    entry_protocol='m3u8_native',
-                    m3u8_id='m3u8-%s' % protocol, fatal=False)
-                # Despite CODECS metadata in m3u8 all video-only formats
-                # are actually video+audio
-                for f in m3u8_formats:
-                    if '_drm/index_' in f['url']:
-                        continue
-                    if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
-                        f['acodec'] = None
-                    formats.append(f)
-            elif format_id in ('mpd', 'dash'):
-                formats.extend(self._extract_mpd_formats(
-                    format_url, video_id, 'mpd-%s' % protocol, fatal=False))
-            elif format_url.startswith('rtmp'):
-                mobj = re.search(
-                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+?))/(?P<playpath>mp4:.+)$',
-                    format_url)
-                if not mobj:
-                    return
-                formats.append({
-                    'format_id': 'rtmp-%s' % format_id,
-                    'ext': 'flv',
-                    'url': mobj.group('url'),
-                    'play_path': mobj.group('playpath'),
-                    'app': mobj.group('app'),
-                    'page_url': url,
-                })
-            else:
-                formats.append({
-                    'url': format_url,
-                    'format_id': '%s-%s' % (format_id, protocol),
-                    'height': int_or_none(self._search_regex(
-                        r'^(\d+)[pP]$', format_id, 'height', default=None)),
-                })
-
-        for format_id, format_dict in (resp.get('streams') or {}).items():
-            add_format(format_id, format_dict)
-        if not formats:
-            streams = self._call_api(
-                'videos/%s/streams.json' % video_id, video_id,
-                'Downloading video streams JSON')
-
-            if 'external' in streams:
-                result.update({
-                    '_type': 'url_transparent',
-                    'url': streams['external']['url'],
-                })
-                return result
-
-            for format_id, stream_dict in streams.items():
-                for protocol, format_dict in stream_dict.items():
-                    add_format(format_id, format_dict, protocol)
-        self._sort_formats(formats)
-
-        result['formats'] = formats
-        return result
-
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
@@ -378,9 +326,9 @@ class VikiChannelIE(VikiBaseIE):
         'info_dict': {
             'id': '50c',
             'title': 'Boys Over Flowers',
-            'description': 'md5:804ce6e7837e1fd527ad2f25420f4d59',
+            'description': 'md5:f08b679c200e1a273c695fe9986f21d7',
         },
-        'playlist_mincount': 71,
+        'playlist_mincount': 51,
     }, {
         'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
         'info_dict': {
@@ -401,33 +349,38 @@ class VikiChannelIE(VikiBaseIE):
         'only_matching': True,
     }]
 
-    _PER_PAGE = 25
+    _video_types = ('episodes', 'movies', 'clips', 'trailers')
+
+    def _entries(self, channel_id):
+        params = {
+            'app': self._APP, 'token': self._token, 'only_ids': 'true',
+            'direction': 'asc', 'sort': 'number', 'per_page': 30
+        }
+        video_types = self._video_types
+        for video_type in video_types:
+            if video_type not in self._video_types:
+                self.report_warning('Unknown video_type: ' + video_type)
+            page_num = 0
+            while True:
+                page_num += 1
+                params['page'] = page_num
+                res = self._call_api(
+                    'containers/{channel_id}/{video_type}.json'.format(**locals()), channel_id, query=params, fatal=False,
+                    note='Downloading %s JSON page %d' % (video_type.title(), page_num))
+
+                for video_id in res.get('response') or []:
+                    yield self.url_result('https://www.viki.com/videos/' + video_id, VikiIE.ie_key(), video_id)
+                if not res.get('more'):
+                    break
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        channel = self._call_api(
-            'containers/%s.json' % channel_id, channel_id,
-            'Downloading channel JSON')
+        channel = self._call_api('containers/%s.json' % channel_id, channel_id, 'Downloading channel JSON')
 
         self._check_errors(channel)
 
-        title = self.dict_selection(channel['titles'], 'en')
-
-        description = self.dict_selection(channel['descriptions'], 'en')
-
-        entries = []
-        for video_type in ('episodes', 'clips', 'movies'):
-            for page_num in itertools.count(1):
-                page = self._call_api(
-                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
-                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
-                    'Downloading %s JSON page #%d' % (video_type, page_num))
-                for video in page['response']:
-                    video_id = video['id']
-                    entries.append(self.url_result(
-                        'https://www.viki.com/videos/%s' % video_id, 'Viki'))
-                if not page['pagination']['next']:
-                    break
-
-        return self.playlist_result(entries, channel_id, title, description)
+        return self.playlist_result(
+            self._entries(channel_id), channel_id,
+            self.dict_selection(channel['titles'], 'en'),
+            self.dict_selection(channel['descriptions'], 'en'))

From 9f10bb8f5b982c3bb31a9368b362be9a364e7cd5 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 4 Feb 2022 18:53:02 +0100
Subject: [PATCH 112/394] cleaning

---
 docs/supportedsites.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8bc14d81dce..727c7df5a7b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1211,6 +1211,7 @@
  - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
+ - **youtube:search_url**: YouTube search URLs with sorting and filter support
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:tab**: YouTube.com tab
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)

From 7606d9759254065c913898e1901fbad6f8216e26 Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sun, 6 Feb 2022 20:11:16 +0400
Subject: [PATCH 113/394] Updated streamff to use externalLink

---
 youtube_dl/extractor/streamff.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
index ebadae7c4f1..0ea035ad20e 100644
--- a/youtube_dl/extractor/streamff.py
+++ b/youtube_dl/extractor/streamff.py
@@ -23,5 +23,5 @@ class StreamFFIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         response = self._download_json("https://streamff.com/api/videos/%s" % video_id, video_id)
-        video_url = "https://streamff.com%s" % response["videoLink"]
-        return self.url_result(video_url, ie="Generic")
+        external_link = response["externalLink"]
+        return self.url_result(external_link, ie="Generic")

From 8105f8ed844304a07806813ee89e461e61316d76 Mon Sep 17 00:00:00 2001
From: Volodymyr <prokopiff@gmail.com>
Date: Mon, 7 Feb 2022 18:30:39 +0200
Subject: [PATCH 114/394] Add extractor for teleportal.ua

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/teleportal.py | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/teleportal.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 50b7cb4a04f..2fbf921ec17 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1227,6 +1227,7 @@
 from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
+from .teleportal import TeleportalIE
 from .telequebec import (
     TeleQuebecIE,
     TeleQuebecSquatIE,
diff --git a/youtube_dl/extractor/teleportal.py b/youtube_dl/extractor/teleportal.py
new file mode 100644
index 00000000000..b254ee3dc77
--- /dev/null
+++ b/youtube_dl/extractor/teleportal.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TeleportalIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?teleportal\.ua(/ua)?/(?P<id>[0-9a-z-/]+)'
+    _TEST = {
+        'url': 'https://teleportal.ua/ua/show/stb/master-cheff/bitva-sezonov/vypusk-3',
+        'md5': '07bd056c45b515fa9cc0202b8403df41',
+        'info_dict': {
+            'id': 'show/stb/master-cheff/bitva-sezonov/vypusk-3',
+            'ext': 'mp4',
+            'title': 'МастерШеф. Битва сезонів 3 випуск: найогидніший випуск сезону!',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': r're:^<p>Не пропустіть.*',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        backend_url = 'https://tp-back.starlight.digital/ua/{}'.format(video_id)
+        metadata = self._download_json(backend_url, video_id)
+        api_metadata = self._download_json('https://vcms-api2.starlight.digital/player-api/{}?referer=https://teleportal.ua/&lang=ua'.format(metadata["hash"]), video_id)
+
+        return {
+            'id': video_id,
+            'title': metadata['title'],
+            'description': metadata['description'],
+            'real_id': metadata['id'],
+            'hash': metadata['hash'],
+            'url': api_metadata['video'][0]['mediaHls'],
+            'thumbnail': api_metadata['video'][0]['poster'],
+            'formats': self._extract_m3u8_formats(api_metadata['video'][0]['mediaHls'], video_id, 'mp4'),
+        }

From 35081ecf00d3c204f29daac04b9f608906cbc8dc Mon Sep 17 00:00:00 2001
From: Volodymyr <prokopiff@gmail.com>
Date: Wed, 9 Feb 2022 21:15:01 +0200
Subject: [PATCH 115/394] Minor fixes

---
 youtube_dl/extractor/teleportal.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/teleportal.py b/youtube_dl/extractor/teleportal.py
index b254ee3dc77..6c976e7f622 100644
--- a/youtube_dl/extractor/teleportal.py
+++ b/youtube_dl/extractor/teleportal.py
@@ -22,15 +22,19 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         backend_url = 'https://tp-back.starlight.digital/ua/{}'.format(video_id)
         metadata = self._download_json(backend_url, video_id)
-        api_metadata = self._download_json('https://vcms-api2.starlight.digital/player-api/{}?referer=https://teleportal.ua/&lang=ua'.format(metadata["hash"]), video_id)
+        api_metadata = self._download_json('https://vcms-api2.starlight.digital/player-api/{}?referer=https://teleportal.ua/&lang=ua'.format(metadata['hash']), video_id)
+
+        try:
+            thumbnail = api_metadata['video'][0]['poster']
+        except (KeyError, IndexError):
+            thumbnail = None
 
         return {
             'id': video_id,
-            'title': metadata['title'],
-            'description': metadata['description'],
-            'real_id': metadata['id'],
-            'hash': metadata['hash'],
-            'url': api_metadata['video'][0]['mediaHls'],
-            'thumbnail': api_metadata['video'][0]['poster'],
+            'title': metadata.get('title'),
+            'description': metadata.get('description'),
+            'real_id': metadata.get('id'),
+            'hash': metadata.get('hash'),
+            'thumbnail': thumbnail,
             'formats': self._extract_m3u8_formats(api_metadata['video'][0]['mediaHls'], video_id, 'mp4'),
         }

From 19e5cd4c323c5122ecbce9ee7a04c18c2bd3b058 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 13 Feb 2022 20:45:28 +0100
Subject: [PATCH 116/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2eade85593c..5b65bb52de4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.03'
+__version__ = '2021.02.13'

From 85fab93d435536cc677e9f4999fd6affa6ded498 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 13 Feb 2022 20:46:11 +0100
Subject: [PATCH 117/394] cleaning

---
 docs/supportedsites.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 727c7df5a7b..52188a995d0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -66,6 +66,7 @@
  - **Arkena**
  - **arte.sky.it**
  - **ArteTV**
+ - **ArteTVCategory**
  - **ArteTVEmbed**
  - **ArteTVPlaylist**
  - **AsianCrush**
@@ -612,6 +613,10 @@
  - **nicknight**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
+ - **NiconicoUser**
+ - **nicovideo:search**: Nico video search
+ - **nicovideo:search:date**: Nico video search, newest first
+ - **nicovideo:search_url**: Nico video search URLs
  - **Nintendo**
  - **njoy**: N-JOY
  - **njoy:embed**
@@ -1143,7 +1148,7 @@
  - **WatchBox**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
- - **wdr:mobile**
+ - **wdr:mobile** (Currently broken)
  - **WDRElefant**
  - **WDRPage**
  - **Webcaster**

From 54e9d638f2dc97a9dfbd9fc3f1871761129fb0cd Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 15 Feb 2022 02:48:44 +0000
Subject: [PATCH 118/394] [Go] Update extractor and tests for current sites and
 API version

Fixes yt-dl version of https://github.com/yt-dlp/yt-dlp/issues/2764
---
 youtube_dl/extractor/go.py | 123 ++++++++++++++++++++++++++++---------
 1 file changed, 95 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 878ba14e655..0a40b5b479a 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -20,10 +20,12 @@ class GoIE(AdobePassIE):
         'abc': {
             'brand': '001',
             'requestor_id': 'ABC',
+            'go': True,
         },
         'freeform': {
             'brand': '002',
             'requestor_id': 'ABCFamily',
+            'go': True,
         },
         'watchdisneychannel': {
             'brand': '004',
@@ -40,6 +42,7 @@ class GoIE(AdobePassIE):
         'disneynow': {
             'brand': '011',
             'resource_id': 'Disney',
+            'go': True,
         },
         'fxnow.fxnetworks': {
             'brand': '025',
@@ -48,15 +51,19 @@ class GoIE(AdobePassIE):
     }
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:
-                            (?:(?P<sub_domain>%s)\.)?go|
-                            (?P<sub_domain_2>abc|freeform|disneynow|fxnow\.fxnetworks)
+                        (?P<sub_domain>
+                            (?:(?:%(sites)s)\.)?go|fxnow\.fxnetworks|
+                            (?:www\.)?(?:%(sites)s)
                         )\.com/
                         (?:
                             (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
                             (?:[^/]+/)*(?P<display_id>[^/?\#]+)
                         )
-                    ''' % '|'.join(list(_SITE_INFO.keys()))
+                    ''' % {
+        'sites':
+            # keep _SITE_INFO in scope
+            (lambda s: '|'.join(re.escape(k) for k in s.keys() if s[k].get('go')))(_SITE_INFO),
+    }
     _TESTS = [{
         'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
         'info_dict': {
@@ -70,14 +77,9 @@ class GoIE(AdobePassIE):
             'skip_download': True,
         },
         'skip': 'This content is no longer available.',
-    }, {
-        'url': 'http://watchdisneyxd.go.com/doraemon',
-        'info_dict': {
-            'title': 'Doraemon',
-            'id': 'SH55574025',
-        },
-        'playlist_mincount': 51,
-    }, {
+    },
+        # Disney XD app withdrawn 2018: 'http://watchdisneyxd.go.com/doraemon',
+        {
         'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
         'info_dict': {
             'id': 'VDKA3609139',
@@ -85,23 +87,37 @@ class GoIE(AdobePassIE):
             'title': 'This Guilty Blood',
             'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
             'age_limit': 14,
+            'duration': 2544,
+            'episode': 'Episode 1',
+            'thumbnail': r're:https?://.*?\.jpg',
+            'season_number': 2,
+            'episode_number': 1,
+            'season': 'Season 2',
+            'series': 'Shadowhunters',
         },
         'params': {
-            'geo_bypass_ip_block': '3.244.239.0/24',
+            'geo_bypass_ip_block': '6.48.97.0/24',
             # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url': 'https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet',
+        'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/13-fight-or-flight',
         'info_dict': {
-            'id': 'VDKA13435179',
+            'id': 'VDKA26139487',
             'ext': 'mp4',
-            'title': 'The Bet',
-            'description': 'md5:c66de8ba2e92c6c5c113c3ade84ab404',
+            'title': 'Fight or Flight',
+            'episode': 'Episode 13',
+            'thumbnail': r're:https?://.*?\.jpg',
+            'series': 'The Rookie',
+            'description': 'md5:8caf91de9806a1c5f79823059fe80267',
+            'duration': 2575,
+            'episode_number': 13,
+            'season_number': 4,
+            'season': 'Season 4',
             'age_limit': 14,
         },
         'params': {
-            'geo_bypass_ip_block': '3.244.239.0/24',
+            'geo_bypass_ip_block': '6.48.97.0/24',
             # m3u8 download
             'skip_download': True,
         },
@@ -112,9 +128,13 @@ class GoIE(AdobePassIE):
             'ext': 'mp4',
             'title': 'First Look: Better Things - Season 2',
             'description': 'md5:fa73584a95761c605d9d54904e35b407',
+            'duration': 161,
+            'series': 'Better Things',
+            'age_limit': 14,
+            'thumbnail': r're:https?://.*?\.jpg',
         },
         'params': {
-            'geo_bypass_ip_block': '3.244.239.0/24',
+            'geo_bypass_ip_block': '6.48.97.0/24',
             # m3u8 download
             'skip_download': True,
         },
@@ -125,6 +145,14 @@ class GoIE(AdobePassIE):
             'ext': 'mp4',
             'title': 'Pilot',
             'description': 'md5:74306df917cfc199d76d061d66bebdb4',
+            'season': 'Season 1',
+            'episode': 'Episode 1',
+            'series': 'Modern Family',
+            'season_number': 1,
+            'duration': 1301,
+            'episode_number': 1,
+            'age_limit': 0,
+            'thumbnail': r're:https?://.*?\.jpg',
         },
         'params': {
             # m3u8 download
@@ -147,17 +175,48 @@ class GoIE(AdobePassIE):
     }, {
         'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
         'only_matching': True,
+    }, {
+        'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
+        'only_matching': True,
+    }, {
+        'url': 'https://disneynow.com/shows/ducktales-disney-channel/season-02/episode-09-the-outlaw-scrooge-mcduck/vdka9680773',
+        'info_dict': {
+            'id': 'VDKA9680773',
+            'ext': 'mp4',
+            'season_number': 2,
+            'title': 'The Outlaw Scrooge McDuck!',
+            'episode': 'Episode 9',
+            'season': 'Season 2',
+            'episode_number': 9,
+            'age_limit': 7,
+            'series': 'DuckTales',
+            'description': 'md5:3d450c1094a5e7544ac094f1e6da2827',
+            'duration': 1348,
+            'thumbnail': r're:https?://.*?\.jpg',
+        },
+        'params': {
+            # AES-128: ffmpeg
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://disneynow.com/shows/ducktales-disney-channel',
+        'info_dict': {
+            'title': 'DuckTales TV Show',
+            'id': 'SH553923438',
+        },
+        'playlist_mincount': 33,
+        'expected_warnings': ['Ignoring subtitle tracks', ],
     }]
 
     def _extract_videos(self, brand, video_id='-1', show_id='-1'):
         display_id = video_id if video_id != '-1' else show_id
         return self._download_json(
-            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
+            'http://api.contents.watchabc.go.com/vp2/ws/contents/4000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
             display_id)['video']
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        sub_domain = mobj.group('sub_domain') or mobj.group('sub_domain_2')
+        sub_domain = re.sub(r'^(?:www\.)?(.+?)(?:\.go)?$', r'\1', mobj.group('sub_domain') or '')
         video_id, display_id = mobj.group('id', 'display_id')
         site_info = self._SITE_INFO.get(sub_domain, {})
         brand = site_info.get('brand')
@@ -181,26 +240,33 @@ def _real_extract(self, url):
                     (
                         # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                         # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
-                        r'data-video-id=["\']*(VDKA\w+)',
+                        r'data-video-id\s*=\s*["\']{,2}(VDKA\w+)',
                         # page.analytics.videoIdCode
                         r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
+                        # This obsolete pattern, if matched, overrides detection of DisneyNow show pages
                         # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
-                        r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)'
+                        # r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)'
                     ), webpage, 'video id', default=video_id)
             if not site_info:
                 brand = self._search_regex(
                     (r'data-brand=\s*["\']\s*(\d+)',
-                     r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
-                    default='004')
+                     r'data-page-brand=\s*["\']\s*(\d+)',
+                     r'"brand"\s*:\s*"(\d+)"'), webpage, 'brand',
+                    # default to a brand that has an active domain
+                    default='011')
                 site_info = next(
-                    si for _, si in self._SITE_INFO.items()
+                    si for si in self._SITE_INFO.values()
                     if si.get('brand') == brand)
             if not video_id:
                 # show extraction works for Disney, DisneyJunior and DisneyXD
                 # ABC and Freeform has different layout
                 show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
                 videos = self._extract_videos(brand, show_id=show_id)
-                show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
+                show_title = (
+                    self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', default=None)
+                    or self._og_search_title(webpage, fatal=False))
+                if show_title:
+                    show_title = re.sub(r'^(?:Watch\s+)?(.+?)(?:\s*\|\s*Disney.*)?$', r'\1', show_title)
                 entries = []
                 for video in videos:
                     entries.append(self.url_result(
@@ -262,7 +328,8 @@ def _real_extract(self, url):
                 if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url):
                     f.update({
                         'format_id': ('%s-' % format_id if format_id else '') + 'SOURCE',
-                        'preference': 1,
+                        'quality': 1,
+                        # 'preference': 1,
                     })
                 else:
                     mobj = re.search(r'/(\d+)x(\d+)/', asset_url)

From 200ce6c1a3aca2b4f96381ef8a76c8699b932089 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 16 Feb 2022 23:18:57 +0000
Subject: [PATCH 119/394] [JSInterp] Improve performance

Pull out some constant regex expressions and change a loop over assignment operators into a regex alternative match
---
 youtube_dl/jsinterp.py | 63 +++++++++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8eaa911cdab..75504db3a99 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -31,8 +31,8 @@ class Nonlocal:
     ('/', operator.truediv),
     ('*', operator.mul),
 ]
-_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
+_ASSIGN_OPERATORS = dict((op + '=', opfunc) for op, opfunc in _OPERATORS)
+_ASSIGN_OPERATORS['='] = lambda cur, right: right
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
@@ -84,6 +84,28 @@ def __repr__(self):
 
 
 class JSInterpreter(object):
+    _EXPR_SPLIT_RE = (
+        r'''(?x)
+            (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
+            (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)'''
+        % {'_NAME_RE': _NAME_RE, })
+    _VARNAME_RE = r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE
+    _ARRAY_REF_RE = r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE
+    _FN_CALL_RE = r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE
+    _MEMBER_REF_RE = (
+        r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*'
+        % _NAME_RE)
+    _FN_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
+    _FN_DEF_RE = (
+        r'(?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}'
+        % _FN_NAME_RE)
+    _ASSIGN_EXPR_RE = (
+        r'''(?x)
+            (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
+            \s*(?P<op>%s)
+            (?P<expr>.*)$'''
+        % (_NAME_RE, '|'.join(re.escape(op) for op in _ASSIGN_OPERATORS.keys())))
+
     def __init__(self, code, objects=None):
         if objects is None:
             objects = {}
@@ -269,9 +291,7 @@ def raise_constructor_error(c):
         for sub_expr in sub_expressions:
             self.interpret_expression(sub_expr, local_vars, allow_recursion)
 
-        for m in re.finditer(r'''(?x)
-                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
-                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+        for m in re.finditer(self._EXPR_SPLIT_RE, expr):
             var = m.group('var1') or m.group('var2')
             start, end = m.span()
             sign = m.group('pre_sign') or m.group('post_sign')
@@ -281,13 +301,10 @@ def raise_constructor_error(c):
                 ret = local_vars[var]
             expr = expr[:start] + json.dumps(ret) + expr[end:]
 
-        for op, opfunc in _ASSIGN_OPERATORS:
-            m = re.match(r'''(?x)
-                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
-                \s*%s
-                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
-            if not m:
-                continue
+        m = re.match(self._ASSIGN_EXPR_RE, expr)
+        if m:
+            op = m.group('op')
+            opfunc = _ASSIGN_OPERATORS[op]
             right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
 
             if m.groupdict().get('index'):
@@ -313,9 +330,7 @@ def raise_constructor_error(c):
         elif expr == 'continue':
             raise JS_Continue()
 
-        var_m = re.match(
-            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
-            expr)
+        var_m = re.match(self._VARNAME_RE, expr)
         if var_m:
             return local_vars[var_m.group('name')]
 
@@ -324,8 +339,7 @@ def raise_constructor_error(c):
         except ValueError:
             pass
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
+        m = re.match(self._ARRAY_REF_RE, expr)
         if m:
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
@@ -350,9 +364,7 @@ def raise_expr_error(where, op, exp):
                 raise_expr_error('right-side', op, expr)
             return opfunc(left_val or 0, right_val)
 
-        m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
-            expr)
+        m = re.match(self._MEMBER_REF_RE, expr)
         if m:
             variable = m.group('var')
             nl = Nonlocal()
@@ -469,7 +481,7 @@ def eval_method():
             else:
                 return eval_method()
 
-        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
+        m = re.match(self._FN_CALL_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
@@ -485,22 +497,17 @@ def eval_method():
             raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
-        _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
             r'''(?x)
                 (?<!this\.)%s\s*=\s*{\s*
                     (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
                 }\s*;
-            ''' % (re.escape(objname), _FUNC_NAME_RE),
+            ''' % (re.escape(objname), self._FN_NAME_RE),
             self.code)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
-        fields_m = re.finditer(
-            r'''(?x)
-                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
-            ''' % _FUNC_NAME_RE,
-            fields)
+        fields_m = re.finditer(self._FN_DEF_RE, fields)
         for f in fields_m:
             argnames = f.group('args').split(',')
             obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))

From 2241472754c9d2e28e8db6feef153f0adba9b762 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 19 Feb 2022 01:30:56 +0000
Subject: [PATCH 120/394] [JSInterp] Use Counter instead of dict comprehension

Py 2.6 FTW
---
 youtube_dl/jsinterp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 75504db3a99..5c401c6c344 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -3,6 +3,7 @@
 import json
 import operator
 import re
+from collections import Counter
 
 from .utils import (
     ExtractorError,
@@ -19,6 +20,7 @@ class Nonlocal:
     pass
 
 
+# tightest first
 _OPERATORS = [
     ('|', operator.or_),
     ('^', operator.xor),
@@ -124,7 +126,7 @@ def _named_object(self, namespace, obj):
     def _separate(expr, delim=',', max_split=None):
         if not expr:
             return
-        counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        counters = Counter()
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         for idx, char in enumerate(expr):
             if char in _MATCHING_PARENS:

From d42e900914056dfd71bbe3d5580616c60cc5febc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 19 Feb 2022 01:34:24 +0000
Subject: [PATCH 121/394] [JSInterp] Introduce JSError with some exception
 tweaks

---
 youtube_dl/jsinterp.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 5c401c6c344..841787b26da 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -41,14 +41,19 @@ class Nonlocal:
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 
 
-class JS_Break(ExtractorError):
+class JSError(ExtractorError):
+    def __init__(self, msg):
+        super(JSError, self).__init__('JS: ' + msg)
+
+
+class JS_Break(JSError):
     def __init__(self):
-        ExtractorError.__init__(self, 'Invalid break')
+        super(JS_Break, self).__init__('Invalid break')
 
 
-class JS_Continue(ExtractorError):
+class JS_Continue(JSError):
     def __init__(self):
-        ExtractorError.__init__(self, 'Invalid continue')
+        super(JS_Continue, self).__init__('Invalid continue')
 
 
 class LocalNameSpace(MutableMapping):
@@ -148,14 +153,15 @@ def _separate(expr, delim=',', max_split=None):
 
     @staticmethod
     def _separate_at_paren(expr, delim):
-        separated = list(JSInterpreter._separate(expr, delim, 1))
-        if len(separated) < 2:
-            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
-        return separated[0][1:].strip(), separated[1].strip()
+        separated = JSInterpreter._separate(expr, delim, 1)
+        try:
+            return next(separated)[1:].strip(), next(separated).strip()
+        except (StopIteration, TypeError):
+            raise JSError('No terminating paren {0} in {1}'.format(delim, expr))
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
-            raise ExtractorError('Recursion limit reached')
+            raise JSError('Recursion limit reached')
 
         sub_statements = list(self._separate(stmt, ';'))
         stmt = (sub_statements or ['']).pop()
@@ -211,7 +217,7 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
 
         m = re.match(r'try\s*', expr)
         if m:
-            if expr[m.end()] == '{':
+            if m.end() < len(expr) and expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
@@ -229,7 +235,7 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
 
         elif md.get('for'):
             def raise_constructor_error(c):
-                raise ExtractorError(
+                raise JSError(
                     'Premature return in the initialization of a for loop in {0!r}'.format(c))
 
             constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
@@ -313,7 +319,7 @@ def raise_constructor_error(c):
                 lvar = local_vars[m.group('out')]
                 idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
                 if not isinstance(idx, int):
-                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
+                    raise JSError('List indices must be integers: %s' % (idx, ))
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -348,7 +354,7 @@ def raise_constructor_error(c):
             return val[idx]
 
         def raise_expr_error(where, op, exp):
-            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+            raise JSError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
 
         for op, opfunc in _OPERATORS:
             separated = list(self._separate(expr, op))
@@ -559,7 +565,7 @@ def resf(args, **kwargs):
             local_vars.update(dict(zip(argnames, args)))
             local_vars.update(kwargs)
             var_stack = LocalNameSpace(local_vars, *global_stack)
-            for stmt in self._separate(code.replace('\n', ''), ';'):
+            for stmt in self._separate(code.replace('\n', ''), ';') or []:
                 ret, should_abort = self.interpret_statement(stmt, var_stack)
                 if should_abort:
                     break

From efae8aaa0aafa4ea3573e6bf3140a91ebc8c7a0c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 20 Feb 2022 20:02:02 +0000
Subject: [PATCH 122/394] [Kaltura] Add live stream support

Fix tests, API return check
---
 youtube_dl/extractor/kaltura.py | 210 ++++++++++++++++++++++----------
 1 file changed, 147 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c731612c4ef..58fdbdb0eeb 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -8,13 +8,16 @@
 from ..compat import (
     compat_urlparse,
     compat_parse_qs,
+    compat_str,
 )
 from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
-    unsmuggle_url,
     smuggle_url,
+    try_get,
+    unsmuggle_url,
+    url_or_none,
 )
 
 
@@ -34,6 +37,7 @@ class KalturaIE(InfoExtractor):
                         )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
                 )
                 '''
+    # See https://www.kaltura.com/api_v3/testmeDoc/
     _SERVICE_URL = 'http://cdnapi.kaltura.com'
     _SERVICE_BASE = '/api_v3/index.php'
     # See https://github.com/kaltura/server/blob/master/plugins/content/caption/base/lib/model/enums/CaptionType.php
@@ -56,6 +60,41 @@ class KalturaIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*/thumbnail/.*',
                 'timestamp': int,
             },
+            'skip': 'The access to this service is forbidden since the specified partner is blocked',
+        },
+        {
+            'url': 'kaltura:956951:1_et8lwckd',
+            'md5': '248d91651534930f8eda6be7484156cc',
+            'info_dict': {
+                'id': '1_et8lwckd',
+                'ext': 'mp4',
+                'title': 'BUS 599 Week 8 Discussion Overview',
+                'upload_date': '20181116',
+                'uploader_id': 'andrea.banto',
+                'timestamp': 1542382513,
+            },
+            'params': {
+                # force reproducible download
+                'format': 'mp4[format_id!^=hls]',
+            },
+        },
+        {
+            # Livestream (Danish Parliament)
+            'url': 'kaltura:2158211:1_24gfa7qq',
+            'info_dict': {
+                'id': '1_24gfa7qq',
+                'ext': 'mp4',
+                'title': 'Folketingets 24/7 TV kanal',
+                'is_live': True,
+                'upload_date': '20180110',
+                'uploader_id': 'jesper@img.dk',
+                'timestamp': 1515581608,
+            },
+            'params': {
+                # live stream
+                'skip_download': True,
+            },
+            'expected_warnings': ['Unable to download f4m manifest'],
         },
         {
             'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
@@ -180,10 +219,23 @@ def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs
             (service_url or self._SERVICE_URL) + self._SERVICE_BASE,
             video_id, query=params, *args, **kwargs)
 
-        status = data if len(actions) == 1 else data[0]
-        if status.get('objectType') == 'KalturaAPIException':
+        def raise_api_error(msg, expected=True):
             raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, status['message']))
+                '%s said: %s' % (self.IE_NAME, msg))
+
+        # general check on result: assume it is either one dict or a list of items,
+        # each either a dict or a list
+        for resp in data if isinstance(data, list) else [data if isinstance(data, dict) else {}]:
+            if isinstance(resp, list):
+                continue
+            resp_type = try_get(resp, lambda x: x['objectType'], compat_str)
+            if resp_type is None:
+                raise_api_error('Empty response from API', expected=False)
+            elif resp_type == 'KalturaAPIException':
+                raise_api_error(
+                    '%s (code %s)' % (
+                        resp.get('message', '--'),
+                        resp.get('code', '--')))
 
         return data
 
@@ -207,7 +259,7 @@ def _get_video_info(self, video_id, partner_id, service_url=None):
                 'entryId': video_id,
                 'service': 'baseentry',
                 'ks': '{1:result:ks}',
-                'responseProfile:fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId',
+                'responseProfile:fields': 'createdAt,dataUrl,duration,liveStreamConfigurations,name,plays,thumbnailUrl,userId',
                 'responseProfile:type': 1,
             },
             {
@@ -288,64 +340,95 @@ def _real_extract(self, url):
         else:
             referrer = None
 
-        def sign_url(unsigned_url):
-            if ks:
-                unsigned_url += '/ks/%s' % ks
-            if referrer:
-                unsigned_url += '?referrer=%s' % referrer
-            return unsigned_url
+        formats = []
+        info_dict = {}
+        if try_get(info, lambda x: x['objectType'], compat_str) == 'KalturaLiveStreamEntry':
+            info_dict['is_live'] = True
+            seen_urls = set()
+            for cnf in try_get(info, lambda x: x['liveStreamConfigurations'], list) or []:
+                m_url = url_or_none(try_get(cnf, lambda x: x['url'], compat_str))
+                if not m_url or m_url in seen_urls:
+                    continue
+                seen_urls.add(m_url)
+                protocol = cnf.get('protocol')
+                # https://cdnapi.kaltura.com/p/2158211/sp/215821100/playManifest/entryId/1_24gfa7qq/protocol/https/format/...
+                # applehttp_to_mc (multicast)??
+                if protocol in ('hls', 'applehttp', 'applehttp_to_mc'):
+                    # ... applehttp/a.m3u8',
+                    # ... applehttp_to_mc/a.m3u8',
+                    formats.extend(self._extract_m3u8_formats(
+                        m_url, entry_id, 'mp4', 'm3u8_native',
+                        m3u8_id=protocol, fatal=False))
+                elif protocol == 'hds':
+                    # ...hds/a.f4m',
+                    formats.extend(self._extract_f4m_formats(
+                        m_url, entry_id, f4m_id=protocol,
+                        m3u8_id=protocol, fatal=False))
+                # elif protocol == 'sl': # SilverLight, really?
+                # ... sl/Manifest
+                elif protocol == 'mpegdash':
+                    # ... mpegdash/manifest.mpd
+                    formats.extend(self._extract_mpd_formats(
+                        m_url, entry_id, mpd_id='dash', fatal=False))
+        else:
 
-        data_url = info['dataUrl']
-        if '/flvclipper/' in data_url:
-            data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)
+            def sign_url(unsigned_url):
+                if ks:
+                    unsigned_url += '/ks/%s' % ks
+                if referrer:
+                    unsigned_url += '?referrer=%s' % referrer
+                return unsigned_url
 
-        formats = []
-        for f in flavor_assets:
-            # Continue if asset is not ready
-            if f.get('status') != 2:
-                continue
-            # Original format that's not available (e.g. kaltura:1926081:0_c03e1b5g)
-            # skip for now.
-            if f.get('fileExt') == 'chun':
-                continue
-            # DRM-protected video, cannot be decrypted
-            if f.get('fileExt') == 'wvm':
-                continue
-            if not f.get('fileExt'):
-                # QT indicates QuickTime; some videos have broken fileExt
-                if f.get('containerFormat') == 'qt':
-                    f['fileExt'] = 'mov'
-                else:
-                    f['fileExt'] = 'mp4'
-            video_url = sign_url(
-                '%s/flavorId/%s' % (data_url, f['id']))
-            format_id = '%(fileExt)s-%(bitrate)s' % f
-            # Source format may not be available (e.g. kaltura:513551:1_66x4rg7o)
-            if f.get('isOriginal') is True and not self._is_valid_url(
-                    video_url, entry_id, format_id):
-                continue
-            # audio-only has no videoCodecId (e.g. kaltura:1926081:0_c03e1b5g
-            # -f mp4-56)
-            vcodec = 'none' if 'videoCodecId' not in f and f.get(
-                'frameRate') == 0 else f.get('videoCodecId')
-            formats.append({
-                'format_id': format_id,
-                'ext': f.get('fileExt'),
-                'tbr': int_or_none(f['bitrate']),
-                'fps': int_or_none(f.get('frameRate')),
-                'filesize_approx': int_or_none(f.get('size'), invscale=1024),
-                'container': f.get('containerFormat'),
-                'vcodec': vcodec,
-                'height': int_or_none(f.get('height')),
-                'width': int_or_none(f.get('width')),
-                'url': video_url,
-            })
-        if '/playManifest/' in data_url:
-            m3u8_url = sign_url(data_url.replace(
-                'format/url', 'format/applehttp'))
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, entry_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
+            data_url = info['dataUrl']
+            if '/flvclipper/' in data_url:
+                data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)
+
+            for f in flavor_assets:
+                # Continue if asset is not ready
+                if f.get('status') != 2:
+                    continue
+                # Original format that's not available (e.g. kaltura:1926081:0_c03e1b5g)
+                # skip for now.
+                if f.get('fileExt') == 'chun':
+                    continue
+                # DRM-protected video, cannot be decrypted
+                if f.get('fileExt') == 'wvm':
+                    continue
+                if not f.get('fileExt'):
+                    # QT indicates QuickTime; some videos have broken fileExt
+                    if f.get('containerFormat') == 'qt':
+                        f['fileExt'] = 'mov'
+                    else:
+                        f['fileExt'] = 'mp4'
+                video_url = sign_url(
+                    '%s/flavorId/%s' % (data_url, f['id']))
+                format_id = '%(fileExt)s-%(bitrate)s' % f
+                # Source format may not be available (e.g. kaltura:513551:1_66x4rg7o)
+                if f.get('isOriginal') is True and not self._is_valid_url(
+                        video_url, entry_id, format_id):
+                    continue
+                # audio-only has no videoCodecId (e.g. kaltura:1926081:0_c03e1b5g
+                # -f mp4-56)
+                vcodec = 'none' if 'videoCodecId' not in f and f.get(
+                    'frameRate') == 0 else f.get('videoCodecId')
+                formats.append({
+                    'format_id': format_id,
+                    'ext': f.get('fileExt'),
+                    'tbr': int_or_none(f['bitrate']),
+                    'fps': int_or_none(f.get('frameRate')),
+                    'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+                    'container': f.get('containerFormat'),
+                    'vcodec': vcodec,
+                    'height': int_or_none(f.get('height')),
+                    'width': int_or_none(f.get('width')),
+                    'url': video_url,
+                })
+            if '/playManifest/' in data_url:
+                m3u8_url = sign_url(data_url.replace(
+                    'format/url', 'format/applehttp'))
+                formats.extend(self._extract_m3u8_formats(
+                    m3u8_url, entry_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
 
         self._sort_formats(formats)
 
@@ -363,7 +446,7 @@ def sign_url(unsigned_url):
                     'ext': caption.get('fileExt') or self._CAPTION_TYPES.get(caption_format) or 'ttml',
                 })
 
-        return {
+        info_dict.update({
             'id': entry_id,
             'title': info['name'],
             'formats': formats,
@@ -374,4 +457,5 @@ def sign_url(unsigned_url):
             'timestamp': info.get('createdAt'),
             'uploader_id': info.get('userId') if info.get('userId') != 'None' else None,
             'view_count': info.get('plays'),
-        }
+        })
+        return info_dict

From 8db78afe671ec15774fdf71756ae471405203586 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 22 Feb 2022 11:24:06 +0000
Subject: [PATCH 123/394] Ignore --external-downloader-args if
 --external-downloader was rejected

... and generate warning
---
 youtube_dl/YoutubeDL.py           | 11 ++++++++++-
 youtube_dl/downloader/__init__.py |  3 +++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef9c..82412baf49d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1906,8 +1906,17 @@ def ensure_dir_exists(path):
 
         if not self.params.get('skip_download', False):
             try:
+                def checked_get_suitable_downloader(info_dict, params):
+                    ed_args = params.get('external_downloader_args')
+                    dler = get_suitable_downloader(info_dict, params)
+                    if ed_args and not params.get('external_downloader_args'):
+                        # external_downloader_args was cleared because external_downloader was rejected
+                        self.report_warning('Requested external downloader cannot be used: '
+                                            'ignoring --external-downloader-args.')
+                    return dler
+
                 def dl(name, info):
-                    fd = get_suitable_downloader(info, self.params)(self, self.params)
+                    fd = checked_get_suitable_downloader(info, self.params)(self, self.params)
                     for ph in self._progress_hooks:
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index d8f2fa34226..d701d629221 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -50,6 +50,9 @@ def _get_suitable_downloader(info_dict, params={}):
         ed = get_external_downloader(external_downloader)
         if ed.can_download(info_dict):
             return ed
+        # Avoid using unwanted args since external_downloader was rejected
+        if params.get('external_downloader_args'):
+            params['external_downloader_args'] = None
 
     protocol = info_dict['protocol']
     if protocol.startswith('m3u8') and info_dict.get('is_live'):

From b6a24ad93d9d4d92356b0c489d3e4a3dd2a0a3f6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 22 Feb 2022 18:47:03 +0000
Subject: [PATCH 124/394] [Anvato] Update extractor for new API; fix tests

---
 youtube_dl/extractor/anvato.py | 84 ++++++++++++++++++++++++++--------
 1 file changed, 64 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index b7398563b35..eb00975c55e 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -17,6 +17,7 @@
     intlist_to_bytes,
     int_or_none,
     strip_jsonp,
+    try_get,
     unescapeHTML,
     unsmuggle_url,
 )
@@ -203,6 +204,7 @@ class AnvatoIE(InfoExtractor):
         'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582'
     }
 
+    _API_PREFIX = 'https://tkx.mp.lura.live/rest/v2/'
     _API_KEY = '3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA'
 
     _ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
@@ -211,15 +213,18 @@ class AnvatoIE(InfoExtractor):
     _TESTS = [{
         # from https://www.boston25news.com/news/watch-humpback-whale-breaches-right-next-to-fishing-boat-near-nh/817484874
         'url': 'anvato:8v9BEynrwx8EFLYpgfOWcG1qJqyXKlRM:4465496',
+        'only_matching': True,
+    }, {
+        # from https://miami.cbslocal.com/2022/02/12/no-appetite-for-new-miami-restaurant-glorifying-castro-communism/
+        'url': 'anvato:5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl:6197559',  # 8v9BEynrwx8EFLYpgfOWcG1qJqyXKlRM:4465496',
         'info_dict': {
-            'id': '4465496',
+            'id': '6197559',
             'ext': 'mp4',
-            'title': 'VIDEO: Humpback whale breaches right next to NH boat',
-            'description': 'VIDEO: Humpback whale breaches right next to NH boat. Footage courtesy: Zach Fahey.',
-            'duration': 22,
-            'timestamp': 1534855680,
-            'upload_date': '20180821',
-            'uploader': 'ANV',
+            'upload_date': '20220209',
+            'uploader': 'CBS',
+            'description': 'CBS4\'s Joel Waldman has more on the backlash Cafe Habana is receiving.',
+            'timestamp': 1644381300,
+            'title': 'Miamians Want No Part Of New Restaurant Set To Open In Brickell That Glorifies Fidel Castro & Communism',
         },
         'params': {
             'skip_download': True,
@@ -228,46 +233,85 @@ class AnvatoIE(InfoExtractor):
         # from https://sanfrancisco.cbslocal.com/2016/06/17/source-oakland-cop-on-leave-for-having-girlfriend-help-with-police-reports/
         'url': 'anvato:DVzl9QRzox3ZZsP9bNu5Li3X7obQOnqP:3417601',
         'only_matching': True,
+    }, {
+        # from https://sanfrancisco.cbslocal.com/2022/02/16/san-francisco-voters-recall-embattled-school-board-members/
+        'url': 'anvato:5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl:6201051',
+        'info_dict': {
+            'id': '6201051',
+            'ext': 'mp4',
+            'upload_date': '20220216',
+            'uploader': 'CBS',
+            'description': 'Voters were successful in their high-profile effort to recall three San Francisco school board members. Anne Makovec reports.',
+            'timestamp': 1645043880,
+            'title': 'Voters Recall San Francisco School Board Members',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def __init__(self, *args, **kwargs):
         super(AnvatoIE, self).__init__(*args, **kwargs)
         self.__server_time = None
 
-    def _server_time(self, access_key, video_id):
+    def _server_time(self, access_key, video_id, server_url=None):
         if self.__server_time is not None:
             return self.__server_time
 
+        if not server_url:
+            server_url = self._API_PREFIX + 'server_time?anvack={ANVACK}'
+
         self.__server_time = int(self._download_json(
-            self._api_prefix(access_key) + 'server_time?anvack=' + access_key, video_id,
+            server_url.format(ANVACK=access_key), video_id,
             note='Fetching server time')['server_time'])
 
         return self.__server_time
 
-    def _api_prefix(self, access_key):
-        return 'https://tkx2-%s.anvato.net/rest/v2/' % ('prod' if 'prod' in access_key else 'stage')
-
     def _get_video_json(self, access_key, video_id):
-        # See et() in anvplayer.min.js, which is an alias of getVideoJSON()
-        video_data_url = self._api_prefix(access_key) + 'mcp/video/%s?anvack=%s' % (video_id, access_key)
-        server_time = self._server_time(access_key, video_id)
+
+        def fix_template_vars(template):
+            return re.sub(r'\{(\{\w+})}', r'\1', template)
+
+        # https://access.mp.lura.live/anvacks/5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl?apikey=3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA
+        access_info = self._download_json(
+            'https://access.mp.lura.live/anvacks/%s?apikey=%s'
+            % (access_key, self._API_KEY),
+            video_id, note='Downloading access details')
+        server_time_url = try_get(access_info, lambda x: x['api']['time'])
+
+        server_time_url = (
+            server_time_url
+            and fix_template_vars(server_time_url).format(ANVACK=access_key))
+        server_time = self._server_time(access_key, video_id, server_time_url)
+
+        video_data_url = access_info['api'].get('video')
+
+        if not video_data_url:
+            # use special knowledge
+            video_data_url = self._API_PREFIX + 'mcp/video/{{VIDEO_ID}}?anvack={{ANVACK}}'
+
+        video_data_url = fix_template_vars(video_data_url).format(ANVACK=access_key, VIDEO_ID=video_id)
+
         input_data = '%d~%s~%s' % (server_time, md5_text(video_data_url), md5_text(server_time))
 
         auth_secret = intlist_to_bytes(aes_encrypt(
             bytes_to_intlist(input_data[:64]), bytes_to_intlist(self._AUTH_KEY)))
 
-        video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
         anvrid = md5_text(time.time() * 1000 * random.random())[:30]
+
+        query = {
+            'X-Anvato-Adst-Auth': base64.b64encode(auth_secret).decode('ascii'),
+            'rtyp': 'fp',
+        }
         api = {
             'anvrid': anvrid,
             'anvts': server_time,
+            'anvstk2': 'default',
         }
-        api['anvstk'] = md5_text('%s|%s|%d|%s' % (
-            access_key, anvrid, server_time,
-            self._ANVACK_TABLE.get(access_key, self._API_KEY)))
-
         return self._download_json(
             video_data_url, video_id, transform_source=strip_jsonp,
+            note='Downloading video details',
+            query=query,
             data=json.dumps({'api': api}).encode('utf-8'))
 
     def _get_anvato_videos(self, access_key, video_id):

From ebf0fcd916db3d462ed9ac75bc2d04381e091ba8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 22 Feb 2022 19:01:44 +0000
Subject: [PATCH 125/394] [SendtoNews] Improve _VALID_URL, fix handling API
 result, add/fix tests

Media links now come in `configuration.sources.src` of playlist item
---
 youtube_dl/extractor/sendtonews.py | 133 ++++++++++++++++++++---------
 1 file changed, 94 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py
index 9d9652949bb..2d63918a36d 100644
--- a/youtube_dl/extractor/sendtonews.py
+++ b/youtube_dl/extractor/sendtonews.py
@@ -5,43 +5,45 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    dict_get,
+    ExtractorError,
     float_or_none,
     parse_iso8601,
     update_url_query,
     int_or_none,
+    determine_ext,
     determine_protocol,
+    strip_or_none,
+    try_get,
     unescapeHTML,
+    urljoin,
 )
 
 
 class SendtoNewsIE(InfoExtractor):
-    _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'
-
-    _TEST = {
+    # TODO handle items with ?fk=XXXX6789&cid=1234 -> SC=XXXX6789-???????-1234
+    _VALID_URL = r'https?://embed\.sendtonews\.com/(?:player\d/embed(?:player|code)\.(?:php|js)|oembed/?)\?.*\bSC=(?P<id>[\w-]+)'
+    _TESTS = [{
         # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
         'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',
         'info_dict': {
             'id': 'GxfCe0Zo7D-175909-5588'
         },
-        'playlist_count': 8,
-        # test the first video only to prevent lengthy tests
-        'playlist': [{
-            'info_dict': {
-                'id': '240385',
-                'ext': 'mp4',
-                'title': 'Indians introduce Encarnacion',
-                'description': 'Indians president of baseball operations Chris Antonetti and Edwin Encarnacion discuss the slugger\'s three-year contract with Cleveland',
-                'duration': 137.898,
-                'thumbnail': r're:https?://.*\.jpg$',
-                'upload_date': '20170105',
-                'timestamp': 1483649762,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+        'playlist_count': 10,
+    }, {
+        'url': 'https://embed.sendtonews.com/player4/embedplayer.php?SC=mq3wIKSb68-1206898-8402&type=single',
+        'info_dict': {
+            'id': '1752278',
+            'ext': 'mp4',
+            'title': 'Las vegas homebuilders had banner sales year in 2021, and other top stories from January 24, 2022.',
+            'description': 'LAS VEGAS HOMEBUILDERS HAD BANNER SALES YEAR IN 2021., and other top stories from January 24, 2022.',
+            'timestamp': 1643063702,
+            'upload_date': '20220124',
+            'thumbnail': r're:https?://.*\.(?:png|jpg)$',
+            'categories': ['Business'],
+            'tags': list,
         },
-    }
+    }]
 
     _URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'
 
@@ -59,17 +61,62 @@ def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         data_url = update_url_query(
-            url.replace('embedplayer.php', 'data_read.php'),
-            {'cmd': 'loadInitial'})
+            re.sub(
+                r'(?P<player>player\d)?(?<!/)/(?P<embed>embed.+?|oembed/)\?',
+                lambda m: '/%s/data_read.php?' % ((m.group('player') or 'player4'), ),
+                url),
+            {'cmd': 'loadInitial', 'type': 'single', })
         playlist_data = self._download_json(data_url, playlist_id)
+        playlist = try_get(playlist_data, lambda x: x['playlistData'][0], (dict, list)) or {}
+        if isinstance(playlist, dict):
+            err = playlist.get('error', 'No or invalid data returned from API')
+            raise ExtractorError(err)
 
         entries = []
-        for video in playlist_data['playlistData'][0]:
-            info_dict = self._parse_jwplayer_data(
-                video['jwconfiguration'],
-                require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True})
+        info_dict = {}
+        for video in playlist:
+            try:
+                err = video.get('error')
+                if err and video.get('S_ID') is not None:
+                    e = ExtractorError(err)
+                    e.msg = err
+                    raise e
+            except AttributeError:
+                continue
+            except ExtractorError as e:
+                self.report_warning(e.msg, playlist_id)
+                continue
+            if 'jwconfiguration' in video:
+                info_dict.update(self._parse_jwplayer_data(
+                    video['jwconfiguration'],
+                    require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True}))
+            elif 'configuration' not in video:
+                continue
+            else:
+                fmt_url = urljoin(
+                    url,
+                    try_get(video, lambda x: x['configuration']['sources']['src']))
+                if not fmt_url:
+                    continue
+                video_id = strip_or_none(video.get('SM_ID') or video['configuration']['mediaid'])
+                title = strip_or_none(video.get('S_headLine') or video['configuration']['title'])
+                if not video_id or not title:
+                    continue
+                ext = determine_ext(fmt_url)
+                if ext == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        fmt_url, playlist_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False)
+                else:
+                    formats = [{
+                        'url': fmt_url,
+                        'ext': ext,
+                        'width': int_or_none(video.get('SM_M_VIDEO_WIDTH')),
+                        'height': int_or_none(video.get('SM_M_VIDEO_HEIGHT')),
+                    }]
+                info_dict['formats'] = formats
 
-            for f in info_dict['formats']:
+            for f in info_dict.get('formats') or []:
                 if f.get('tbr'):
                     continue
                 tbr = int_or_none(self._search_regex(
@@ -83,23 +130,31 @@ def _real_extract(self, url):
             self._sort_formats(info_dict['formats'], ('tbr', 'height', 'width', 'format_id'))
 
             thumbnails = []
-            if video.get('thumbnailUrl'):
-                thumbnails.append({
-                    'id': 'normal',
-                    'url': video['thumbnailUrl'],
-                })
-            if video.get('smThumbnailUrl'):
+            for tn_id, tn in (('poster', video['configuration'].get('poster')),
+                              ('normal', video.get('thumbnailUrl')),
+                              ('small', video.get('smThumbnailUrl'))):
+                tn = urljoin(url, tn)
+                if not tn:
+                    continue
                 thumbnails.append({
-                    'id': 'small',
-                    'url': video['smThumbnailUrl'],
+                    'id': tn_id,
+                    'url': tn,
                 })
             info_dict.update({
-                'title': video['S_headLine'].strip(),
-                'description': unescapeHTML(video.get('S_fullStory')),
+                'id': video_id,
+                'title': title,
+                'description': unescapeHTML(dict_get(video, ('S_fullStory', 'S_shortSummary'))),
                 'thumbnails': thumbnails,
-                'duration': float_or_none(video.get('SM_length')),
+                'duration': float_or_none(
+                    dict_get(video, ('SM_length', 'SM_M_LENGTH'))
+                    or video['configuration'].get('duration')),
                 'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
+                'tags': [t for t in video.get('S_tags', '').split(',') if t],
+                'categories': [c for c in video.get('S_category', '').split(',') if c],
             })
             entries.append(info_dict)
 
+        if len(entries) == 1:
+            entries[0]['display_id'] = playlist_id
+            return entries[0]
         return self.playlist_result(entries, playlist_id)

From 015954f21a369aa63101b3e9aca0ee06a340f93a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 22 Feb 2022 22:20:11 +0000
Subject: [PATCH 126/394] [core]  Make default upload_/release_date a
 compat_str

Ensures CBSLocal download test passes in Python 2 (pre-release from PR #29698)
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef9c..69736acffa6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1529,7 +1529,7 @@ def sanitize_numeric_fields(info):
                 # see http://bugs.python.org/issue1646728)
                 try:
                     upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
-                    info_dict[date_key] = upload_date.strftime('%Y%m%d')
+                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
                 except (ValueError, OverflowError, OSError):
                     pass
 

From 108737d60067d92357358765724090b8bccdbe39 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 23 Feb 2022 02:55:38 +0000
Subject: [PATCH 127/394] [CBSLocal] Handle rehosted legacy Anvato video,
 add/fix tests

Update regex for Anvato player JSON
---
 youtube_dl/extractor/anvato.py   | 46 +++++++++++---------
 youtube_dl/extractor/cbslocal.py | 74 ++++++++++++++++++--------------
 2 files changed, 67 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index eb00975c55e..ae4c5f2e969 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -207,7 +207,9 @@ class AnvatoIE(InfoExtractor):
     _API_PREFIX = 'https://tkx.mp.lura.live/rest/v2/'
     _API_KEY = '3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA'
 
-    _ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
+    _ANVP_RE = (
+        r'<script[^>]*>[^<]*?\bAnvatoPlayer\s*\(\s*["\w]+\s*\)\s*\.\s*init\s*\(\s*(?P<anvp>{[^<]+?})\s*\);',
+        r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1')
     _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
 
     _TESTS = [{
@@ -381,26 +383,28 @@ def _get_anvato_videos(self, access_key, video_id):
     @staticmethod
     def _extract_urls(ie, webpage, video_id):
         entries = []
-        for mobj in re.finditer(AnvatoIE._ANVP_RE, webpage):
-            anvplayer_data = ie._parse_json(
-                mobj.group('anvp'), video_id, transform_source=unescapeHTML,
-                fatal=False)
-            if not anvplayer_data:
-                continue
-            video = anvplayer_data.get('video')
-            if not isinstance(video, compat_str) or not video.isdigit():
-                continue
-            access_key = anvplayer_data.get('accessKey')
-            if not access_key:
-                mcp = anvplayer_data.get('mcp')
-                if mcp:
-                    access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
-                        mcp.lower())
-            if not access_key:
-                continue
-            entries.append(ie.url_result(
-                'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
-                video_id=video))
+        anvp_res = AnvatoIE._ANVP_RE
+        for anvp_re in anvp_res if isinstance(anvp_res, (list, tuple, )) else (anvp_res, ):
+            for mobj in re.finditer(anvp_re, webpage):
+                anvplayer_data = ie._parse_json(
+                    mobj.group('anvp'), video_id, transform_source=unescapeHTML,
+                    fatal=False)
+                if not anvplayer_data:
+                    continue
+                video = anvplayer_data.get('video')
+                if not isinstance(video, compat_str) or not video.isdigit():
+                    continue
+                access_key = anvplayer_data.get('accessKey')
+                if not access_key:
+                    mcp = anvplayer_data.get('mcp')
+                    if mcp:
+                        access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
+                            mcp.lower())
+                if not access_key:
+                    continue
+                entries.append(ie.url_result(
+                    'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
+                    video_id=video))
         return entries
 
     def _extract_anvato_videos(self, webpage, video_id):
diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 3b7e1a8b9f6..f8bffab27aa 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -5,6 +5,7 @@
 from .sendtonews import SendtoNewsIE
 from ..compat import compat_urlparse
 from ..utils import (
+    merge_dicts,
     parse_iso8601,
     unified_timestamp,
 )
@@ -14,6 +15,8 @@ class CBSLocalIE(AnvatoIE):
     _VALID_URL_BASE = r'https?://[a-z]+\.cbslocal\.com/'
     _VALID_URL = _VALID_URL_BASE + r'video/(?P<id>\d+)'
 
+    _OLD_ANVATO_KEY = 'anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67'
+
     _TESTS = [{
         'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
         'info_dict': {
@@ -30,10 +33,6 @@ class CBSLocalIE(AnvatoIE):
             },
             'categories': [
                 'Stations\\Spoken Word\\WCBSTV',
-                'Syndication\\AOL',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\Yahoo',
                 'Content\\News',
                 'Content\\News\\Local News',
             ],
@@ -42,12 +41,21 @@ class CBSLocalIE(AnvatoIE):
         'params': {
             'skip_download': True,
         },
+        'expected_warnings': ('Failed to download m3u8 information', ),
     }]
 
     def _real_extract(self, url):
+
         mcp_id = self._match_id(url)
-        return self.url_result(
-            'anvato:anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67:' + mcp_id, 'Anvato', mcp_id)
+        webpage = self._download_webpage(url, mcp_id)
+
+        json_ld = self._search_json_ld(webpage, mcp_id, fatal=False) or {}
+        json_ld.pop('url', None)
+
+        return merge_dicts(
+            self._extract_anvato_videos(webpage, mcp_id)
+            or self.url_result(self._OLD_ANVATO_KEY + ':' + mcp_id, 'Anvato', mcp_id),
+            json_ld)
 
 
 class CBSLocalArticleIE(AnvatoIE):
@@ -56,30 +64,25 @@ class CBSLocalArticleIE(AnvatoIE):
     _TESTS = [{
         # Anvato backend
         'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
-        'md5': 'f0ee3081e3843f575fccef901199b212',
+        'only_matching': True
+    }, {
+        'url': 'https://losangeles.cbslocal.com/2022/02/16/rams-super-bowl-parade-to-take-place-wednesday/',
+        'md5': '36bdac3fb24ec8a6d7790218a0357b08',
         'info_dict': {
-            'id': '3401037',
+            'id': '6201053',
             'ext': 'mp4',
-            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
-            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
-            'thumbnail': 're:^https?://.*',
-            'timestamp': 1463440500,
-            'upload_date': '20160516',
+            'display_id': 'rams-super-bowl-parade-to-take-place-wednesday',
+            'upload_date': '20220216',
             'uploader': 'CBS',
-            'subtitles': {
-                'en': 'mincount:5',
-            },
+            'description': 'Jeff Nguyen is live from outside the L.A. Memorial Coliseum where fans cheered on the Los Angeles Rams.',
+            'timestamp': 1645044990,
+            'title': 'Rams Fans Gather Outside The LA Memorial Coliseum',
             'categories': [
-                'Stations\\Spoken Word\\KCBSTV',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\AOL',
-                'Syndication\\Yahoo',
-                'Syndication\\Tribune',
-                'Syndication\\Curb.tv',
-                'Content\\News'
+                'Stations\\Spoken Word\\KCALTV',
+                'Content\\News',
+                'Content\\Top Story',
             ],
-            'tags': ['CBS 2 News Evening'],
+            'tags': ['KCAL 9 News Afternoon'],
         },
     }, {
         # SendtoNews embed
@@ -92,18 +95,24 @@ class CBSLocalArticleIE(AnvatoIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Redirects to CBS News home page',
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
+        json_ld = self._search_json_ld(webpage, display_id, fatal=False) or {}
+        json_ld.pop('url', None)
+
         sendtonews_url = SendtoNewsIE._extract_url(webpage)
         if sendtonews_url:
-            return self.url_result(
+            result = self.url_result(
                 compat_urlparse.urljoin(url, sendtonews_url),
                 ie=SendtoNewsIE.ie_key())
+            return merge_dicts(result, json_ld)
 
+        # returns a dict, or raises
         info_dict = self._extract_anvato_videos(webpage, display_id)
 
         timestamp = unified_timestamp(self._html_search_regex(
@@ -111,9 +120,10 @@ def _real_extract(self, url):
             'released date', default=None)) or parse_iso8601(
             self._html_search_meta('uploadDate', webpage))
 
-        info_dict.update({
-            'display_id': display_id,
-            'timestamp': timestamp,
-        })
-
-        return info_dict
+        return merge_dicts(
+            info_dict,
+            json_ld,
+            {
+                'display_id': display_id,
+                'timestamp': timestamp,
+            })

From 57aad1ae4bd4ae27e9363cd3e2031ef8486d3a8b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 24 Feb 2022 09:16:16 +0000
Subject: [PATCH 128/394] [CPAC] Add extractor for Canadian Parliament

CPACIE: single episode
CPACPlaylistIE: playlists and searches
---
 youtube_dl/extractor/cpac.py       | 148 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   4 +
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/cpac.py

diff --git a/youtube_dl/extractor/cpac.py b/youtube_dl/extractor/cpac.py
new file mode 100644
index 00000000000..22741152c64
--- /dev/null
+++ b/youtube_dl/extractor/cpac.py
@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    str_or_none,
+    try_get,
+    unified_timestamp,
+    update_url_query,
+    urljoin,
+)
+
+# compat_range
+try:
+    if callable(xrange):
+        range = xrange
+except (NameError, TypeError):
+    pass
+
+
+class CPACIE(InfoExtractor):
+    IE_NAME = 'cpac'
+    _VALID_URL = r'https?://(?:www\.)?cpac\.ca/(?P<fr>l-)?episode\?id=(?P<id>[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12})'
+    _TEST = {
+        # 'url': 'http://www.cpac.ca/en/programs/primetime-politics/episodes/65490909',
+        'url': 'https://www.cpac.ca/episode?id=fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+        'md5': 'e46ad699caafd7aa6024279f2614e8fa',
+        'info_dict': {
+            'id': 'fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+            'ext': 'mp4',
+            'upload_date': '20220215',
+            'title': 'News Conference to Celebrate National Kindness Week – February 15, 2022',
+            'description': 'md5:466a206abd21f3a6f776cdef290c23fb',
+            'timestamp': 1644901200,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'hls_prefer_native': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if '/l-episode?' in url else 'en'
+
+        content = self._download_json(
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/episode/index.xml&crafterSite=cpacca&id=' + video_id,
+            video_id)
+        video_url = try_get(content, lambda x: x['page']['details']['videoUrl'], compat_str)
+        formats = []
+        if video_url:
+            content = content['page']
+            title = str_or_none(content['details']['title_%s_t' % (url_lang, )])
+            formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', ext='mp4')
+            for fmt in formats:
+                # prefer language to match URL
+                fmt_lang = fmt.get('language')
+                if fmt_lang == url_lang:
+                    fmt['language_preference'] = 10
+                elif not fmt_lang:
+                    fmt['language_preference'] = -1
+                else:
+                    fmt['language_preference'] = -10
+
+        self._sort_formats(formats)
+
+        category = str_or_none(content['details']['category_%s_t' % (url_lang, )])
+
+        def is_live(v_type):
+            return (v_type == 'live') if v_type is not None else None
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'description': str_or_none(content['details'].get('description_%s_t' % (url_lang, ))),
+            'timestamp': unified_timestamp(content['details'].get('liveDateTime')),
+            'category': [category] if category else None,
+            'thumbnail': urljoin(url, str_or_none(content['details'].get('image_%s_s' % (url_lang, )))),
+            'is_live': is_live(content['details'].get('type')),
+        }
+
+
+class CPACPlaylistIE(InfoExtractor):
+    IE_NAME = 'cpac:playlist'
+    _VALID_URL = r'(?i)https?://(?:www\.)?cpac\.ca/(?:program|search|(?P<fr>emission|rechercher))\?(?:[^&]+&)*?(?P<id>(?:id=\d+|programId=\d+|key=[^&]+))'
+
+    _TESTS = [{
+        'url': 'https://www.cpac.ca/program?id=6',
+        'info_dict': {
+            'id': 'id=6',
+            'title': 'Headline Politics',
+            'description': 'Watch CPAC’s signature long-form coverage of the day’s pressing political events as they unfold.',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.cpac.ca/search?key=hudson&type=all&order=desc',
+        'info_dict': {
+            'id': 'key=hudson',
+            'title': 'hudson',
+        },
+        'playlist_count': 22,
+    }, {
+        'url': 'https://www.cpac.ca/search?programId=50',
+        'info_dict': {
+            'id': 'programId=50',
+            'title': '50',
+        },
+        'playlist_count': 9,
+    }, {
+        'url': 'https://www.cpac.ca/emission?id=6',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cpac.ca/rechercher?key=hudson&type=all&order=desc',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if any(x in url for x in ('/emission?', '/rechercher?')) else 'en'
+        pl_type, list_type = ('program', 'itemList') if any(x in url for x in ('/program?', '/emission?')) else ('search', 'searchResult')
+        api_url = (
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/%s/index.xml&crafterSite=cpacca&%s'
+            % (pl_type, video_id, ))
+        content = self._download_json(api_url, video_id)
+        entries = []
+        total_pages = int_or_none(try_get(content, lambda x: x['page'][list_type]['totalPages']), default=1)
+        for page in range(1, total_pages + 1):
+            if page > 1:
+                api_url = update_url_query(api_url, {'page': '%d' % (page, ), })
+                content = self._download_json(
+                    api_url, video_id,
+                    note='Downloading continuation - %d' % (page, ),
+                    fatal=False)
+
+            for item in try_get(content, lambda x: x['page'][list_type]['item'], list) or []:
+                episode_url = urljoin(url, try_get(item, lambda x: x['url_%s_s' % (url_lang, )]))
+                if episode_url:
+                    entries.append(episode_url)
+
+        return self.playlist_result(
+            (self.url_result(entry) for entry in entries),
+            playlist_id=video_id,
+            playlist_title=try_get(content, lambda x: x['page']['program']['title_%s_t' % (url_lang, )]) or video_id.split('=')[-1],
+            playlist_description=try_get(content, lambda x: x['page']['program']['description_%s_t' % (url_lang, )]),
+        )
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 50b7cb4a04f..9474c85193e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -254,6 +254,10 @@
 from .condenast import CondeNastIE
 from .contv import CONtvIE
 from .corus import CorusIE
+from .cpac import (
+    CPACIE,
+    CPACPlaylistIE,
+)
 from .cracked import CrackedIE
 from .crackle import CrackleIE
 from .crooksandliars import CrooksAndLiarsIE

From 231ad81bbc0d26edfbb42063a5575dfd588a9b98 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 24 Feb 2022 13:13:24 +0000
Subject: [PATCH 129/394] minimising lines changed

---
 youtube_dl/extractor/myspass.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 39e56f7df5d..f540c52ee68 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -36,13 +36,12 @@ def _real_extract(self, url):
         video_url = xpath_text(metadata, 'url_flv', 'download url', True)
         video_id_int = int(video_id)
 
-        regex_check = re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url)
-        if regex_check:
-            for group in regex_check.groups():
-                group_int = int(group)
-                if group_int > video_id_int:
-                    video_url = video_url.replace(
-                        group, compat_str(group_int // video_id_int))
+        grps = re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url)
+        for group in grps.groups() if grps else []:
+            group_int = int(group)
+            if group_int > video_id_int:
+                video_url = video_url.replace(
+                    group, compat_str(group_int // video_id_int))
 
         return {
             'id': video_id,

From a02f0288e278ccc6461599dc0536703340dae7e2 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Thu, 24 Feb 2022 15:39:26 +0100
Subject: [PATCH 130/394] cleaning

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 52188a995d0..4e182732437 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -201,6 +201,8 @@
  - **CONtv**
  - **Corus**
  - **Coub**
+ - **cpac**
+ - **cpac:playlist**
  - **Cracked**
  - **Crackle**
  - **CrooksAndLiars**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5b65bb52de4..40f4a0b73ab 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.13'
+__version__ = '2021.02.24'

From 3918da628d1848ed3ec0fe98c242b5c25defd6b1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 24 Feb 2022 23:52:47 +0000
Subject: [PATCH 131/394] Extract further fields; implement some review
 comments

---
 youtube_dl/extractor/teleportal.py | 83 ++++++++++++++++++++++++------
 1 file changed, 66 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/teleportal.py b/youtube_dl/extractor/teleportal.py
index 6c976e7f622..3becc264838 100644
--- a/youtube_dl/extractor/teleportal.py
+++ b/youtube_dl/extractor/teleportal.py
@@ -2,39 +2,88 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    float_or_none,
+    parse_iso8601,
+    str_or_none,
+    str_to_int,
+    try_get,
+    url_or_none,
+)
 
 
 class TeleportalIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?teleportal\.ua(/ua)?/(?P<id>[0-9a-z-/]+)'
+    _VALID_URL = r'https?://(?:www\.)?teleportal\.ua/(?:ua/)?(?P<id>[0-9a-z-]+(?:/[0-9a-z-]+)*)'
     _TEST = {
         'url': 'https://teleportal.ua/ua/show/stb/master-cheff/bitva-sezonov/vypusk-3',
-        'md5': '07bd056c45b515fa9cc0202b8403df41',
+        # no permanent check on file contents as HLS may vary
         'info_dict': {
             'id': 'show/stb/master-cheff/bitva-sezonov/vypusk-3',
             'ext': 'mp4',
             'title': 'МастерШеф. Битва сезонів 3 випуск: найогидніший випуск сезону!',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'description': r're:^<p>Не пропустіть.*',
-        }
+            'display_id': '2618466',
+            'description': 'md5:4179bcc3a12edfa2f655888cd741ac09',
+            'timestamp': 1644102480,
+            'upload_date': '20220205',
+            'thumbnail': r're:^https?://.+\.jpg$',
+            'release_timestamp': 1643994000,
+            'duration': 11254.0,
+            'series_id': '20632',
+            'series': 'МастерШеф. Битва сезонів 3 випуск: найогидніший випуск сезону!',
+            'season': 'Битва сезонів',
+            'episode': 'Найогидніший випуск сезону!',
+            'episode_num': 3,
+            'categories': ['Шоу'],
+        },
+        'params': {
+            'hls_prefer_native': True,
+            # 'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        backend_url = 'https://tp-back.starlight.digital/ua/{}'.format(video_id)
-        metadata = self._download_json(backend_url, video_id)
-        api_metadata = self._download_json('https://vcms-api2.starlight.digital/player-api/{}?referer=https://teleportal.ua/&lang=ua'.format(metadata['hash']), video_id)
+        backend_url = 'https://tp-back.starlight.digital/ua/' + video_id
+        series_metadata = self._download_json(backend_url, video_id) or {}
+        title = series_metadata['title']
+        _hash = series_metadata.get('hash', '')
+        api_url = 'https://vcms-api2.starlight.digital/player-api/' + _hash
+        api_metadata = self._download_json(
+            api_url, video_id,
+            query={
+                'referer': 'https://teleportal.ua/',
+                'lang': 'ua',
+            }
+        )
+        video_info = api_metadata['video'][0]
+        formats = []
+        for media in ('mediaHlsNoAdv', 'mediaHls'):
+            media = url_or_none(try_get(video_info, lambda x: x[media]))
+            if not media:
+                continue
+            formats.extend(self._extract_m3u8_formats(media, video_id, 'mp4', fatal=False))
+            break
+        self._sort_formats(formats)
 
-        try:
-            thumbnail = api_metadata['video'][0]['poster']
-        except (KeyError, IndexError):
-            thumbnail = None
+        thumbnail = url_or_none(video_info.get('poster'))
+        category = series_metadata.get('typeTitle')
 
         return {
             'id': video_id,
-            'title': metadata.get('title'),
-            'description': metadata.get('description'),
-            'real_id': metadata.get('id'),
-            'hash': metadata.get('hash'),
+            'title': title,
+            'formats': formats,
+            'description': clean_html(series_metadata.get('description')) or series_metadata.get('seoDescription'),
+            'display_id': str_or_none(video_info.get('vcmsId')),
+            'hash': _hash,
             'thumbnail': thumbnail,
-            'formats': self._extract_m3u8_formats(api_metadata['video'][0]['mediaHls'], video_id, 'mp4'),
+            'timestamp': parse_iso8601(video_info.get('time_upload_video'), delimiter=' '),
+            'release_timestamp': parse_iso8601(video_info.get('publishDate'), delimiter=' '),
+            'duration': float_or_none(video_info.get('duration')),
+            'series_id': str_or_none(series_metadata.get('id')),
+            'series': series_metadata.get('title'),
+            'season': video_info.get('seasonName') or series_metadata.get('seasonGallery', {}).get('title'),
+            'episode': video_info.get('name'),
+            'episode_num': str_to_int(series_metadata.get('seriesTitle')),
+            'categories': [category] if category else None,
         }

From 76144ccf0f4483e5e162080a2e6cbabcb24dda57 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 26 Feb 2022 13:03:51 +0000
Subject: [PATCH 132/394] [Filmarkivet] Add extractor for Filmarkivet.se

https://github.com/yt-dlp/yt-dlp/issues/2885
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/filmark.py    | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/filmark.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 50b7cb4a04f..5d10b91f3aa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -372,6 +372,7 @@
     FilmOnIE,
     FilmOnChannelIE,
 )
+from .filmark import FilmarkIE
 from .filmweb import FilmwebIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/filmark.py b/youtube_dl/extractor/filmark.py
new file mode 100644
index 00000000000..2eca90b525e
--- /dev/null
+++ b/youtube_dl/extractor/filmark.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    merge_dicts,
+    parse_iso8601,
+)
+
+
+class FilmarkIE(InfoExtractor):
+    IE_NAME = 'Filmarkivet.se'
+    _VALID_URL = r'https?://www\.filmarkivet\.se/movies/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.filmarkivet.se/movies/paris-d-moll/',
+        'md5': 'df02cadc719dcc63d43288366f037754',
+        'info_dict': {
+            'id': 'paris-d-moll',
+            'ext': 'mp4',
+            'upload_date': '20200602',
+            'title': 'Paris d-moll',
+            'description': 'md5:319e37ea5542293db37e1e13072fe330',
+            'timestamp': 1591092663,
+        }
+    },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = (
+            self._html_search_regex(r'<title\b[^>]*>\s*([^<]*)</title\b', webpage, '<title>', fatal=False)
+            or self._og_search_title(webpage, fatal=False)
+            or self._html_search_meta('twitter:title', webpage, default=''))
+        title = re.split(r'(\s*-)?\s*[Ff]ilmarkivet', title, 1)[0] or self._generic_title(url)
+
+        ld_json_info = self._search_json_ld(webpage, video_id, default={})
+
+        jwp_info = self._extract_jwplayer_data(webpage, video_id, require_title=False)
+
+        description = (
+            self._og_search_description(webpage)
+            or self._html_search_meta('twitter:description', webpage, default=None))
+        timestamp = parse_iso8601(self._og_search_property('updated_time', webpage, default=None))
+
+        return merge_dicts(
+            jwp_info, ld_json_info, {
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+            })

From 76f2b63a26a5dd416b2760d9b3e23155f7f27f92 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Mon, 25 Oct 2021 10:38:32 +0900
Subject: [PATCH 133/394] [extractor/mtv] fix mgid extraction

Fixes: #29814, #30139
---
 youtube_dl/extractor/mtv.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5a5205c0e98..142945b09ec 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -290,7 +290,17 @@ def _extract_mgid(self, webpage):
             main_container = self._extract_child_with_type(data, 'MainContainer')
             ab_testing = self._extract_child_with_type(main_container, 'ABTesting')
             video_player = self._extract_child_with_type(ab_testing or main_container, 'VideoPlayer')
-            mgid = video_player['props']['media']['video']['config']['uri']
+            if video_player:
+                mgid = video_player['props']['media']['video']['config']['uri']
+            else:
+                flex_wrapper = self._extract_child_with_type(ab_testing or main_container, 'FlexWrapper')
+                auth_suite_wrapper = self._extract_child_with_type(flex_wrapper, 'AuthSuiteWrapper')
+                player = self._extract_child_with_type(auth_suite_wrapper or flex_wrapper, 'Player')
+                if player:
+                    mgid = player['props']['videoDetail']['mgid']
+
+        if not mgid:
+            raise ExtractorError('Could not extract mgid')
 
         return mgid
 

From 2fe0d70b6e3d301cdd2c48e8293325cae5345301 Mon Sep 17 00:00:00 2001
From: Sipherdrakon <64430430+Sipherdrakon@users.noreply.github.com>
Date: Wed, 24 Nov 2021 03:01:49 -0500
Subject: [PATCH 134/394] Update test for SouthPark

Original PR: https://github.com/yt-dlp/yt-dlp/pull/1713
Authored by: Sipherdrakon
---
 youtube_dl/extractor/southpark.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 0774da06e40..eb784291f05 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -6,19 +6,18 @@
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark(?:\.cc|studios)\.com/(?:clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark(?:\.cc|studios)\.com/((?:video-)?clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
 
     _TESTS = [{
-        'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured',
+        'url': 'https://southpark.cc.com/video-clips/d7wr06/south-park-you-all-agreed-to-counseling',
         'info_dict': {
-            'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30',
             'ext': 'mp4',
-            'title': 'South Park|Bat Daded',
-            'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
-            'timestamp': 1112760000,
-            'upload_date': '20050406',
+            'title': 'You All Agreed to Counseling',
+            'description': 'Kenny, Cartman, Stan, and Kyle visit Mr. Mackey and ask for his help getting Mrs. Nelson to come back. Mr. Mackey reveals the only way to get things back to normal is to get the teachers vaccinated.',
+            'timestamp': 1615352400,
+            'upload_date': '20210310',
         },
     }, {
         'url': 'http://southpark.cc.com/collections/7758/fan-favorites/1',
@@ -40,11 +39,11 @@ def _get_feed_query(self, uri):
 
 class SouthParkEsIE(SouthParkIE):
     IE_NAME = 'southpark.cc.com:español'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/es/episodios/(?P<id>.+?)(\?|#|$))'
     _LANG = 'es'
 
     _TESTS = [{
-        'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
+        'url': 'http://southpark.cc.com/es/episodios/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
         'info_dict': {
             'title': 'Cartman Consigue Una Sonda Anal',
             'description': 'Cartman Consigue Una Sonda Anal',

From 1cd4bb0d829db8e7c36090aee564d34fe420135c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Feb 2022 04:52:38 +0000
Subject: [PATCH 135/394] [mtv] Get timestamp from <airDate> if not found in
 <pubDate> of feed XML

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 142945b09ec..36caf9f1ba5 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -157,7 +157,7 @@ def _get_video_info(self, itemdoc, use_hls=True):
 
         description = strip_or_none(xpath_text(itemdoc, 'description'))
 
-        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate'))
+        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate') or xpath_text(itemdoc, 'airDate'))
 
         title_el = None
         if title_el is None:

From 81d6b10f48d255b762d0e7a08612d0d510ea0234 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Feb 2022 04:54:24 +0000
Subject: [PATCH 136/394] [utils] timeconvert() should return int

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58de..363aaaf1c7a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2074,7 +2074,7 @@ def timeconvert(timestr):
     timetuple = email.utils.parsedate_tz(timestr)
     if timetuple is not None:
         timestamp = email.utils.mktime_tz(timetuple)
-    return timestamp
+    return int(timestamp) if timestamp is not None else timestamp
 
 
 def sanitize_filename(s, restricted=False, is_id=False):

From 44f73880fed2cd1f4095fb1af7641bf82563ab4a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Feb 2022 04:55:00 +0000
Subject: [PATCH 137/394] [SouthPark] Fix tests again

---
 youtube_dl/extractor/southpark.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index eb784291f05..3277b49f152 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -13,10 +13,11 @@ class SouthParkIE(MTVServicesInfoExtractor):
     _TESTS = [{
         'url': 'https://southpark.cc.com/video-clips/d7wr06/south-park-you-all-agreed-to-counseling',
         'info_dict': {
+            'id': '31929ad5-8269-11eb-8774-70df2f866ace',
             'ext': 'mp4',
             'title': 'You All Agreed to Counseling',
             'description': 'Kenny, Cartman, Stan, and Kyle visit Mr. Mackey and ask for his help getting Mrs. Nelson to come back. Mr. Mackey reveals the only way to get things back to normal is to get the teachers vaccinated.',
-            'timestamp': 1615352400,
+            'timestamp': 1615377600,
             'upload_date': '20210310',
         },
     }, {
@@ -68,6 +69,7 @@ class SouthParkDeIE(SouthParkIE):
             'timestamp': 1380160800,
             'upload_date': '20130926',
         },
+        'skip': 'Geo-restricted',
     }, {
         # non-ASCII characters in initial URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
@@ -76,6 +78,7 @@ class SouthParkDeIE(SouthParkIE):
             'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
         },
         'playlist_count': 3,
+        'skip': 'Geo-restricted',
     }, {
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
@@ -84,6 +87,7 @@ class SouthParkDeIE(SouthParkIE):
             'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
         },
         'playlist_count': 3,
+        'skip': 'Geo-restricted',
     }, {
         'url': 'http://www.southpark.de/collections/2476/superhero-showdown/1',
         'only_matching': True,
@@ -102,6 +106,7 @@ class SouthParkNlIE(SouthParkIE):
             'description': 'Stan is addicted to the new Terrance and Phillip mobile game.',
         },
         'playlist_mincount': 3,
+        'skip': 'Geo-restricted',
     }]
 
 
@@ -117,6 +122,7 @@ class SouthParkDkIE(SouthParkIE):
             'description': 'Butters is convinced he\'s living in a virtual reality.',
         },
         'playlist_mincount': 3,
+        'skip': 'Geo-restricted',
     }, {
         'url': 'http://www.southparkstudios.dk/collections/2476/superhero-showdown/1',
         'only_matching': True,

From ab2df904e0d0341b3f0d799ab77019b9fbf83983 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Sun, 15 Aug 2021 14:14:30 +0900
Subject: [PATCH 138/394] [extractor/facebook] Improve metadata extraction

* add another data structure for video extraction
* modify metadata extraction due to site change
---
 youtube_dl/extractor/facebook.py | 147 ++++++++++++++++++++++---------
 1 file changed, 106 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 04650af39a2..786a958498b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -23,6 +23,7 @@
     int_or_none,
     js_to_json,
     limit_length,
+    merge_dicts,
     parse_count,
     qualities,
     sanitized_Request,
@@ -82,8 +83,9 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '274175099429670',
             'ext': 'mp4',
-            'title': 're:^Asif Nawab Butt posted a video',
+            'title': 'Asif Nawab Butt',
             'uploader': 'Asif Nawab Butt',
+            'uploader_id': '100005115464446',
             'upload_date': '20140506',
             'timestamp': 1399398998,
             'thumbnail': r're:^https?://.*',
@@ -137,18 +139,20 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20160223',
             'uploader': 'Barack Obama',
         },
+        'skip': 'Gif on giphy.com gone',
     }, {
         # have 1080P, but only up to 720p in swf params
         # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
-        'md5': '9571fae53d4165bbbadb17a94651dcdc',
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
-            'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',
+            'title': 'Holocaust survivor becomes US citizen',
+            'description': 'She survived the holocaust — and years later, she’s getting her citizenship so she can vote for Hillary Clinton http://cnn.it/2eERh5f',
             'timestamp': 1477818095,
             'upload_date': '20161030',
             'uploader': 'CNN',
+            'uploader_id': '5550296508',
             'thumbnail': r're:^https?://.*',
             'view_count': int,
         },
@@ -159,10 +163,12 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '1417995061575415',
             'ext': 'mp4',
-            'title': 'md5:1db063d6a8c13faa8da727817339c857',
+            'title': 'Ukrainian Scientists Worldwide | Довгоочікуване відео',
+            'description': 'Довгоочікуване відео',
             'timestamp': 1486648217,
             'upload_date': '20170209',
             'uploader': 'Yaroslav Korpan',
+            'uploader_id': '100000948048708',
         },
         'params': {
             'skip_download': True,
@@ -181,16 +187,20 @@ class FacebookIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        # TODO: parse this webpage
+        'skip': 'Cannot parse webpage well',
     }, {
         # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
         'info_dict': {
-            'id': '1396382447100162',
+            'id': '202882990186699',
             'ext': 'mp4',
-            'title': 'md5:19a428bbde91364e3de815383b54a235',
+            'title': 'birb (O v O\") | Hello? Yes your uber ride is here',
+            'description': 'md5:963dee8a667a2b49f2059cf7ab54fe55',
             'timestamp': 1486035494,
             'upload_date': '20170202',
             'uploader': 'Elisabeth Ahtn',
+            'uploader_id': '100013949973717',
         },
         'params': {
             'skip_download': True,
@@ -231,8 +241,12 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '359649331226507',
             'ext': 'mp4',
-            'title': '#ESLOne VoD - Birmingham Finals Day#1 Fnatic vs. @Evil Geniuses',
+            'title': 'Fnatic vs. EG - Group A - Opening Match - ESL One Birmingham Day 1',
+            'description': '#ESLOne VoD - Birmingham Finals Day#1 Fnatic vs. @Evil Geniuses',
+            'timestamp': 1527084179,
+            'upload_date': '20180523',
             'uploader': 'ESL One Dota 2',
+            'uploader_id': '234218833769558',
         },
         'params': {
             'skip_download': True,
@@ -255,6 +269,7 @@ class FacebookIE(InfoExtractor):
             'id': '10157667649866271',
         },
         'playlist_count': 3,
+        'skip': 'Requires logging in',
     }, {
         # data.nodes[].comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://m.facebook.com/Alliance.Police.Department/posts/4048563708499330',
@@ -279,6 +294,7 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20161122',
             'timestamp': 1479793574,
         },
+        'skip': 'No video',
     }, {
         # data.video.creation_story.attachments[].media
         'url': 'https://www.facebook.com/watch/live/?v=1823658634322275',
@@ -290,6 +306,18 @@ class FacebookIE(InfoExtractor):
         },
         'playlist_count': 1,
         'skip': 'Requires logging in',
+    }, {
+        # data.node.comet_sections.content.story.attachments[].styles.attachment.media
+        'url': 'https://www.facebook.com/groups/352666925484718/permalink/1112505706167499/',
+        'info_dict': {
+            'id': '1080405282779948',
+            'ext': 'mp4',
+            'title': 'Best Chinese Song',
+            'timestamp': 1641225430,
+            'upload_date': '20220103',
+            'uploader_id': '1847991063',
+            'uploader': 'Best Chinese Song',
+        },
     }]
     _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
     _api_config = {
@@ -381,6 +409,63 @@ def _extract_from_url(self, url, video_id):
         webpage = self._download_webpage(
             url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
 
+        def extract_metadata(webpage):
+            video_title = self._html_search_regex(
+                r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
+                'title', default=None)
+            if not video_title:
+                video_title = self._html_search_regex(
+                    r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
+                    webpage, 'alternative title', default=None)
+            if not video_title:
+                video_title = self._html_search_meta(
+                    ['og:title', 'twitter:title', 'description'],
+                    webpage, 'title', default=None)
+            if video_title:
+                video_title = limit_length(video_title, 80)
+            else:
+                video_title = 'Facebook video #%s' % video_id
+            description = self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'],
+                webpage, 'description', default=None)
+            uploader = clean_html(get_element_by_id(
+                'fbPhotoPageAuthorName', webpage)) or self._search_regex(
+                r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader',
+                default=None) or self._og_search_title(webpage, default=None)
+            timestamp = int_or_none(self._search_regex(
+                r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
+                'timestamp', default=None))
+            thumbnail = self._html_search_meta(
+                ['og:image', 'twitter:image'], webpage, 'thumbnail', default=None)
+            # some webpages contain unretrievable thumbnail urls
+            # like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1
+            # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
+            if thumbnail and not re.search(r'\.(?:jpg|png)', thumbnail):
+                thumbnail = None
+            view_count = parse_count(self._search_regex(
+                [r'\bviewCount\s*:\s*["\']([\d,.]+)', r'"video_view_count"\s*:\s*([\d,.]+)'],
+                webpage, 'view count', default=None))
+            info_dict = {
+                'title': video_title,
+                'description': description,
+                'uploader': uploader,
+                'timestamp': timestamp,
+                'thumbnail': thumbnail,
+                'view_count': view_count,
+            }
+            info_json_ld = self._search_json_ld(webpage, video_id, default={})
+            if not info_json_ld:
+                # expecting title, description, uploader, comment_count and like_count
+                info_json_ld = self._search_json_ld(
+                    webpage, video_id, expected_type='SocialMediaPosting', default={})
+                # if we already have a non-generic title, honor that
+                if info_dict['title'] != 'Facebook video #%s' % video_id:
+                    info_json_ld['title'] = info_dict['title']
+            if info_json_ld.get('title'):
+                info_json_ld['title'] = limit_length(
+                    re.sub(r'\s*\|\s*Facebook$', '', info_json_ld['title']), 80)
+            return merge_dicts(info_json_ld, info_dict)
+
         video_data = None
 
         def extract_video_data(instances):
@@ -492,7 +577,10 @@ def parse_attachment(attachment, key='media'):
                         lambda x: x['attachments']
                     ], list) or []
                     for attachment in attachments:
-                        attachment = try_get(attachment, lambda x: x['style_type_renderer']['attachment'], dict)
+                        attachment = try_get(attachment, [
+                            lambda x: x['style_type_renderer']['attachment'],
+                            lambda x: x['styles']['attachment']
+                        ], dict) or {}
                         ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
                         for n in ns:
                             parse_attachment(n)
@@ -513,7 +601,15 @@ def parse_attachment(attachment, key='media'):
                     if not entries:
                         parse_graphql_video(video)
 
-                return self.playlist_result(entries, video_id)
+                if len(entries) > 1:
+                    return self.playlist_result(entries, video_id)
+
+                video_info = entries[0]
+                webpage_info = extract_metadata(webpage)
+                # honor precise duration in video info
+                if video_info.get('duration'):
+                    webpage_info['duration'] = video_info['duration']
+                return merge_dicts(webpage_info, video_info)
 
         if not video_data:
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
@@ -633,43 +729,12 @@ def parse_attachment(attachment, key='media'):
 
         process_formats(formats)
 
-        video_title = self._html_search_regex(
-            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
-            'title', default=None)
-        if not video_title:
-            video_title = self._html_search_regex(
-                r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', default=None)
-        if not video_title:
-            video_title = self._html_search_meta(
-                'description', webpage, 'title', default=None)
-        if video_title:
-            video_title = limit_length(video_title, 80)
-        else:
-            video_title = 'Facebook video #%s' % video_id
-        uploader = clean_html(get_element_by_id(
-            'fbPhotoPageAuthorName', webpage)) or self._search_regex(
-            r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader',
-            default=None) or self._og_search_title(webpage, fatal=False)
-        timestamp = int_or_none(self._search_regex(
-            r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
-            'timestamp', default=None))
-        thumbnail = self._html_search_meta(['og:image', 'twitter:image'], webpage)
-
-        view_count = parse_count(self._search_regex(
-            r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
-            default=None))
-
         info_dict = {
             'id': video_id,
-            'title': video_title,
             'formats': formats,
-            'uploader': uploader,
-            'timestamp': timestamp,
-            'thumbnail': thumbnail,
-            'view_count': view_count,
             'subtitles': subtitles,
         }
+        info_dict.update(extract_metadata(webpage))
 
         return info_dict
 

From 078f26831711ef9b73aba6cadf6947187cc479a5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Feb 2022 13:31:42 +0000
Subject: [PATCH 139/394] [common] Extract author from SocialMediaPosting in
 ld+json

---
 youtube_dl/extractor/common.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 797c35fd52d..7ea45d4a706 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1273,9 +1273,21 @@ def extract_interaction_statistic(e):
                     continue
                 info[count_key] = interaction_count
 
+        def extract_author(e):
+            author = e.get('author')
+            if not author:
+                return
+            info.update({
+                # author can be an instance of 'Organization' or 'Person' types.
+                # both types can have 'name' property(inherited from 'Thing' type). [1]
+                # however some websites are using 'Text' type instead.
+                # 1. https://schema.org/VideoObject
+                'uploader': author.get('name') if isinstance(author, dict) else author if isinstance(author, compat_str) else None,
+            })
+
         def extract_video_object(e):
             assert e['@type'] == 'VideoObject'
-            author = e.get('author')
+            extract_author(e)
             info.update({
                 'url': url_or_none(e.get('contentUrl')),
                 'title': unescapeHTML(e.get('name')),
@@ -1283,11 +1295,6 @@ def extract_video_object(e):
                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
                 'duration': parse_duration(e.get('duration')),
                 'timestamp': unified_timestamp(e.get('uploadDate')),
-                # author can be an instance of 'Organization' or 'Person' types.
-                # both types can have 'name' property(inherited from 'Thing' type). [1]
-                # however some websites are using 'Text' type instead.
-                # 1. https://schema.org/VideoObject
-                'uploader': author.get('name') if isinstance(author, dict) else author if isinstance(author, compat_str) else None,
                 'filesize': float_or_none(e.get('contentSize')),
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),
@@ -1332,6 +1339,8 @@ def extract_video_object(e):
                         'title': unescapeHTML(e.get('headline')),
                         'description': unescapeHTML(e.get('articleBody')),
                     })
+                elif item_type == 'SocialMediaPosting':
+                    extract_author(e)
                 elif item_type == 'VideoObject':
                     extract_video_object(e)
                     if expected_type is None:

From ecd14c832de1663a2b4d06827ba584be30167bda Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Feb 2022 13:33:28 +0000
Subject: [PATCH 140/394] [extractor/facebook] Update extraction improvements
 and fix tests

* avoid crashing in parse_attachment() on invalid attachment
* ignore empty results in <meta> search
---
 youtube_dl/extractor/facebook.py | 40 +++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 786a958498b..a3f1bd32555 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -83,7 +83,7 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '274175099429670',
             'ext': 'mp4',
-            'title': 'Asif Nawab Butt',
+            'title': 'Asif',
             'uploader': 'Asif Nawab Butt',
             'uploader_id': '100005115464446',
             'upload_date': '20140506',
@@ -173,6 +173,7 @@ class FacebookIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'This video is only available for registered users',
     }, {
         'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
         'info_dict': {
@@ -188,15 +189,16 @@ class FacebookIE(InfoExtractor):
             'skip_download': True,
         },
         # TODO: parse this webpage
-        'skip': 'Cannot parse webpage well',
+        'skip': 'This video is only available for registered users (and cannot parse webpage well)',
     }, {
         # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
         'info_dict': {
             'id': '202882990186699',
             'ext': 'mp4',
+
             'title': 'birb (O v O\") | Hello? Yes your uber ride is here',
-            'description': 'md5:963dee8a667a2b49f2059cf7ab54fe55',
+            'description': 'md5:f122681cf504d04be12bb2bdc66c81fd',
             'timestamp': 1486035494,
             'upload_date': '20170202',
             'uploader': 'Elisabeth Ahtn',
@@ -308,15 +310,17 @@ class FacebookIE(InfoExtractor):
         'skip': 'Requires logging in',
     }, {
         # data.node.comet_sections.content.story.attachments[].styles.attachment.media
-        'url': 'https://www.facebook.com/groups/352666925484718/permalink/1112505706167499/',
+        # gone : 'https://www.facebook.com/groups/352666925484718/permalink/1112505706167499/',
+        'url': 'https://www.facebook.com/groups/471738070722101/permalink/471799617382613/',
         'info_dict': {
-            'id': '1080405282779948',
+            'id': '101173872166725',
             'ext': 'mp4',
-            'title': 'Best Chinese Song',
-            'timestamp': 1641225430,
-            'upload_date': '20220103',
-            'uploader_id': '1847991063',
-            'uploader': 'Best Chinese Song',
+            'title': 'md5:0ac174e60ed9d9a5eec588343f6a889c',
+            'description': 'md5:80efdff1371fa5d92758df2fb1b97bfd',
+            'upload_date': '20210517',
+            'timestamp': 1621247381,
+            'uploader': 'md5:34165f897c692f92b13c2cc1eddd8245',
+            'uploader_id': '100957655521680',
         },
     }]
     _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
@@ -417,15 +421,23 @@ def extract_metadata(webpage):
                 video_title = self._html_search_regex(
                     r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                     webpage, 'alternative title', default=None)
+
+            # _html_search_meta() finds '' ahead of later non-empty results
+            def html_search_meta_non_empty(patterns, html, name, **kwargs):
+                for p in patterns:
+                    result = self._html_search_meta(p, html, name, **kwargs)
+                    if result:
+                        return result
+
             if not video_title:
-                video_title = self._html_search_meta(
+                video_title = html_search_meta_non_empty(
                     ['og:title', 'twitter:title', 'description'],
                     webpage, 'title', default=None)
             if video_title:
                 video_title = limit_length(video_title, 80)
             else:
                 video_title = 'Facebook video #%s' % video_id
-            description = self._html_search_meta(
+            description = html_search_meta_non_empty(
                 ['description', 'og:description', 'twitter:description'],
                 webpage, 'description', default=None)
             uploader = clean_html(get_element_by_id(
@@ -435,7 +447,7 @@ def extract_metadata(webpage):
             timestamp = int_or_none(self._search_regex(
                 r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
                 'timestamp', default=None))
-            thumbnail = self._html_search_meta(
+            thumbnail = html_search_meta_non_empty(
                 ['og:image', 'twitter:image'], webpage, 'thumbnail', default=None)
             # some webpages contain unretrievable thumbnail urls
             # like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1
@@ -562,7 +574,7 @@ def parse_graphql_video(video):
                     entries.append(info)
 
                 def parse_attachment(attachment, key='media'):
-                    media = attachment.get(key) or {}
+                    media = try_get(attachment, lambda x: x[key], dict) or {}
                     if media.get('__typename') == 'Video':
                         return parse_graphql_video(media)
 

From d509040d4ee64e2787dcd148a2fa412a337fbe41 Mon Sep 17 00:00:00 2001
From: hatienl0i261299 <loiht.b17vt220@stu.ptit.edu.vn>
Date: Sun, 27 Feb 2022 21:24:55 +0700
Subject: [PATCH 141/394] Feat: update new version for zingmp3

---
 youtube_dl/extractor/zingmp3.py | 163 +++++++++++++++++++++++---------
 1 file changed, 117 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 207c04f5ea5..75d64ee9a92 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -1,23 +1,52 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import datetime
+import hashlib
+import hmac
+import re
+from urllib.parse import urlencode
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     int_or_none,
+    try_get,
 )
 
 
 class ZingMp3BaseIE(InfoExtractor):
-    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?:%s)/[^/]+/(?P<id>\w+)\.html'
+    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>(?:%s))/[^/]+/(?P<id>\w+)(?:\.html|\?)'
     _GEO_COUNTRIES = ['VN']
+    _IS_UPDATE_COOKIES = False
+    _DOMAIN = 'https://zingmp3.vn'
+    _SLUG_API = {
+        'bai-hat': '/api/v2/page/get/song',
+        'embed': '/api/v2/page/get/song',
+        'video-clip': '/api/v2/page/get/video',
+        'playlist': '/api/v2/page/get/playlist',
+        'album': '/api/v2/page/get/playlist',
+        'lyric': '/api/v2/lyric/get/lyric',
+        'song_streaming': '/api/v2/song/get/streaming',
+    }
+
+    _TIMESTAMP = str(int(datetime.datetime.now().timestamp()))
+    _API_KEY = '88265e23d4284f25963e6eedac8fbfa3'
+    _SECRET_KEY = b'2aa2d1c561e809b267f3638c4a307aab'
+
+    def _extract_item(self, item, song_id, type_url, fatal):
+        item_id = item.get('encodeId') or song_id
+        title = item.get('title') or item.get('alias')
 
-    def _extract_item(self, item, fatal):
-        item_id = item['id']
-        title = item.get('name') or item['title']
+        if type_url == 'video-clip':
+            source = item.get('streaming')
+        else:
+            api = self.get_api_with_signature(name_api=self._SLUG_API.get('song_streaming'), param={
+                'ctime': self._TIMESTAMP,
+                'id': item_id})
+            source = self.download_json(api, item_id).get('data')
 
         formats = []
-        for k, v in (item.get('source') or {}).items():
+        for k, v in (source or {}).items():
             if not v:
                 continue
             if k in ('mp4', 'hls'):
@@ -36,30 +65,34 @@ def _extract_item(self, item, fatal):
                                 r'^(\d+)p', res, 'resolution', default=None)),
                         })
             else:
-                formats.append({
-                    'ext': 'mp3',
-                    'format_id': k,
-                    'tbr': int_or_none(k),
-                    'url': self._proto_relative_url(v),
-                    'vcodec': 'none',
-                })
+                if v != 'VIP':
+                    formats.append({
+                        'ext': 'mp3',
+                        'format_id': k,
+                        'tbr': int_or_none(k),
+                        'url': self._proto_relative_url(v),
+                        'vcodec': 'none',
+                    })
         if not formats:
             if not fatal:
                 return
-            msg = item['msg']
+            msg = item.get('msg')
             if msg == 'Sorry, this content is not available in your country.':
                 self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
-            raise ExtractorError(msg, expected=True)
         self._sort_formats(formats)
 
-        subtitles = None
         lyric = item.get('lyric')
-        if lyric:
-            subtitles = {
-                'origin': [{
-                    'url': lyric,
-                }],
-            }
+        if not lyric:
+            api = self.get_api_with_signature(name_api=self._SLUG_API.get("lyric"), param={
+                'ctime': self._TIMESTAMP,
+                'id': item_id})
+            info_lyric = self.download_json(api, item_id)
+            lyric = try_get(info_lyric, lambda x: x['data']['file'])
+        subtitles = {
+            'origin': [{
+                'url': lyric,
+            }],
+        } if lyric else None
 
         album = item.get('album') or {}
 
@@ -67,30 +100,65 @@ def _extract_item(self, item, fatal):
             'id': item_id,
             'title': title,
             'formats': formats,
-            'thumbnail': item.get('thumbnail'),
+            'thumbnail': item.get('thumbnail') or item.get('thumbnailM'),
             'subtitles': subtitles,
             'duration': int_or_none(item.get('duration')),
             'track': title,
-            'artist': item.get('artists_names'),
+            'artist': item.get('artistsNames') or item.get('artists_names'),
             'album': album.get('name') or album.get('title'),
-            'album_artist': album.get('artists_names'),
+            'album_artist': album.get('artistsNames') or album.get('artists_names'),
         }
 
     def _real_extract(self, url):
-        page_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url.replace('://zingmp3.vn/', '://mp3.zing.vn/'),
-            page_id, query={'play_song': 1})
-        data_path = self._search_regex(
-            r'data-xml="([^"]+)', webpage, 'data path')
-        return self._process_data(self._download_json(
-            'https://mp3.zing.vn/xhr' + data_path, page_id)['data'])
+        mobj = re.match(self._VALID_URL, url)
+
+        song_id = mobj.group('id')
+        type_url = mobj.group('type')
+
+        api = self.get_api_with_signature(name_api=self._SLUG_API[type_url], param={
+            'ctime': self._TIMESTAMP,
+            'id': song_id})
+        info = self.download_json(api, song_id)
+        return self._process_data(info.get('data'), song_id, type_url)
+
+    def download_json(self, api, video_id):
+        # For the first time making the request, we need to make one more temp request to update the cookie
+        # and the next time we don't need to do it.
+        # But if download with cookieFile, so we don't need to make one more temp request
+        if self._downloader.params.get('cookiefile'):
+            self._IS_UPDATE_COOKIES = True
+
+        if not self._IS_UPDATE_COOKIES:
+            self._download_json(api, video_id=video_id, note='Update cookies for request')
+            self._IS_UPDATE_COOKIES = True
+
+        return self._download_json(api, video_id=video_id)
+
+    def get_api_with_signature(self, name_api, param):
+        text = ''
+        for k, v in param.items():
+            text += f'{k}={v}'
+
+        sha256 = self.get_hash256(text)
+        data = {
+            'ctime': param.get('ctime'),
+            'apiKey': self._API_KEY,
+            'sig': self.get_hmac512(f'{name_api}{sha256}')
+        }
+        data.update(param)
+        return f'{self._DOMAIN}{name_api}?{urlencode(data)}'
+
+    def get_hash256(self, string):
+        return hashlib.sha256(string.encode('utf-8')).hexdigest()
+
+    def get_hmac512(self, string):
+        return hmac.new(self._SECRET_KEY, string.encode('utf-8'), hashlib.sha512).hexdigest()
 
 
 class ZingMp3IE(ZingMp3BaseIE):
-    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip'
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed'
     _TESTS = [{
-        'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
+        'url': 'https://zingmp3.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'md5': 'ead7ae13693b3205cbc89536a077daed',
         'info_dict': {
             'id': 'ZWZB9WAB',
@@ -109,7 +177,7 @@ class ZingMp3IE(ZingMp3BaseIE):
             'album_artist': 'Bảo Thy',
         },
     }, {
-        'url': 'https://mp3.zing.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html',
+        'url': 'https://zingmp3.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html',
         'md5': 'e9c972b693aa88301ef981c8151c4343',
         'info_dict': {
             'id': 'ZO8ZF7C7',
@@ -120,15 +188,18 @@ class ZingMp3IE(ZingMp3BaseIE):
             'track': 'Sương Hoa Đưa Lối',
             'artist': 'K-ICM, RYO',
         },
+    }, {
+        'url': 'https://zingmp3.vn/embed/song/ZWZEI76B?start=false',
+        'only_matching': True,
     }, {
         'url': 'https://zingmp3.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'only_matching': True,
     }]
     IE_NAME = 'zingmp3'
-    IE_DESC = 'mp3.zing.vn'
+    IE_DESC = 'zingmp3.vn'
 
-    def _process_data(self, data):
-        return self._extract_item(data, True)
+    def _process_data(self, data, song_id, type_url):
+        return self._extract_item(data, song_id, type_url, True)
 
 
 class ZingMp3AlbumIE(ZingMp3BaseIE):
@@ -140,7 +211,7 @@ class ZingMp3AlbumIE(ZingMp3BaseIE):
             'id': 'ZWZBWDAF',
             'title': 'Lâu Đài Tình Ái',
         },
-        'playlist_count': 10,
+        'playlist_count': 9,
     }, {
         'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
@@ -150,12 +221,12 @@ class ZingMp3AlbumIE(ZingMp3BaseIE):
     }]
     IE_NAME = 'zingmp3:album'
 
-    def _process_data(self, data):
+    def _process_data(self, data, song_id, type_url):
         def entries():
-            for item in (data.get('items') or []):
-                entry = self._extract_item(item, False)
+            for item in (try_get(data, lambda x: x['song']['items']) or []):
+                entry = self._extract_item(item, song_id, type_url, False)
                 if entry:
                     yield entry
-        info = data.get('info') or {}
-        return self.playlist_result(
-            entries(), info.get('id'), info.get('name') or info.get('title'))
+
+        album_id = data.get('id') or data.get('encodeId')
+        return self.playlist_result(entries(), album_id, data.get('name') or data.get('title'))

From c28622047b6b418f60b71cd11763b636a2a859c2 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 28 Feb 2022 22:20:29 +0100
Subject: [PATCH 142/394] cleaning

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4e182732437..8d372c5c141 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -43,6 +43,8 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **Alsace20TV**
+ - **Alsace20TVEmbed**
  - **Amara**
  - **AMCNetworks**
  - **AmericasTestKitchen**
@@ -108,6 +110,7 @@
  - **bfmtv:live**
  - **BibelTV**
  - **Bigflix**
+ - **Bigo**
  - **Bild**: Bild.de
  - **BiliBili**
  - **BilibiliAudio**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 40f4a0b73ab..4b4389cc404 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.24'
+__version__ = '2021.02.28'

From 1208f35f6c529a4cb47b4ee26fa0fb712bef8083 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 28 Feb 2022 22:24:20 +0100
Subject: [PATCH 143/394] cleaning

---
 docs/supportedsites.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8d372c5c141..786f61a19c6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -297,6 +297,7 @@
  - **fc2**
  - **fc2:embed**
  - **Fczenit**
+ - **Filmarkivet.se**
  - **filmon**
  - **filmon:channel**
  - **Filmweb**

From fb4bf37cfe1ce25121dfb17d5a55039f62e03774 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Mon, 28 Feb 2022 22:24:54 +0100
Subject: [PATCH 144/394] cleaning

---
 docs/supportedsites.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 786f61a19c6..ba242f9d14d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -948,6 +948,7 @@
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
  - **TeleMB**
+ - **Teleportal**
  - **TeleQuebec**
  - **TeleQuebecEmission**
  - **TeleQuebecLive**

From d799c405010e50f75e253ace218de9c4be8744a9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Mar 2022 08:11:50 +0000
Subject: [PATCH 145/394] [extractor/ceskatelevize] Back-port extractor from
 yt-dlp

* back-port extractor with traverse_obj() shim, removing CeskaTelevizePoradyIE
* follow redirect URL
* support liveBroadcast and videobonusDetail in __NEXT__ data
* id must be compat_str
* return single video for singleton playlist
* fix/add tests
---
 youtube_dl/extractor/ceskatelevize.py | 277 ++++++++++++++++++--------
 youtube_dl/extractor/extractors.py    |   5 +-
 2 files changed, 199 insertions(+), 83 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 7cb4efb746b..59c0ae0a456 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,70 +12,243 @@
     ExtractorError,
     float_or_none,
     sanitized_Request,
-    unescapeHTML,
-    update_url_query,
+    str_or_none,
     urlencode_postdata,
     USER_AGENTS,
 )
 
+try:
+    from ..utils import traverse_obj
+except ImportError:
+    from ..compat import (compat_str, compat_collections_abc)
+    from ..utils import int_or_none
+
+    def traverse_obj(obj, *path_list, **kw):
+        ''' Traverse nested list/dict/tuple'''
+
+        # parameter defaults
+        default = kw.get('default')
+        expected_type = kw.get('expected_type')
+        get_all = kw.get('get_all', True)
+        casesense = kw.get('casesense', True)
+        is_user_input = kw.get('is_user_input', False)
+        traverse_string = kw.get('traverse_string', False)
+
+        def variadic(x, allowed_types=(compat_str, bytes)):
+            return x if isinstance(x, compat_collections_abc.Iterable) and not isinstance(x, allowed_types) else (x,)
+
+        def listish(l):
+            return isinstance(l, (list, tuple))
+
+        def from_iterable(iterables):
+            # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
+            for it in iterables:
+                for element in it:
+                    yield element
+
+        class Nonlocal:
+            pass
+        nl = Nonlocal()
+
+        if not casesense:
+            _lower = lambda k: (k.lower() if isinstance(k, compat_str) else k)
+            path_list = (map(_lower, variadic(path)) for path in path_list)
+
+        def _traverse_obj(obj, path, _current_depth=0):
+            path = tuple(variadic(path))
+            for i, key in enumerate(path):
+                if obj is None:
+                    return None
+                if listish(key):
+                    obj = [_traverse_obj(obj, sub_key, _current_depth) for sub_key in key]
+                    key = Ellipsis
+                if key is Ellipsis:
+                    obj = (obj.values() if isinstance(obj, dict)
+                           else obj if listish(obj)
+                           else compat_str(obj) if traverse_string else [])
+                    _current_depth += 1
+                    nl.depth = max(nl.depth, _current_depth)
+                    return [_traverse_obj(inner_obj, path[i + 1:], _current_depth) for inner_obj in obj]
+                elif callable(key):
+                    if listish(obj):
+                        obj = enumerate(obj)
+                    elif isinstance(obj, dict):
+                        obj = obj.items()
+                    else:
+                        if not traverse_string:
+                            return None
+                        obj = str(obj)
+                    _current_depth += 1
+                    nl.depth = max(nl.depth, _current_depth)
+                    return [_traverse_obj(v, path[i + 1:], _current_depth) for k, v in obj if key(k)]
+                elif isinstance(obj, dict) and not (is_user_input and key == ':'):
+                    obj = (obj.get(key) if casesense or (key in obj)
+                           else next((v for k, v in obj.items() if _lower(k) == key), None))
+                else:
+                    if is_user_input:
+                        key = (int_or_none(key) if ':' not in key
+                               else slice(*map(int_or_none, key.split(':'))))
+                        if key == slice(None):
+                            return _traverse_obj(obj, tuple([Ellipsis] + list(path[i + 1:])), _current_depth)
+                    if not isinstance(key, (int, slice)):
+                        return None
+                    if not listish(obj):
+                        if not traverse_string:
+                            return None
+                        obj = compat_str(obj)
+                    try:
+                        obj = obj[key]
+                    except IndexError:
+                        return None
+            return obj
+
+        if isinstance(expected_type, type):
+            type_test = lambda val: val if isinstance(val, expected_type) else None
+        elif expected_type is not None:
+            type_test = expected_type
+        else:
+            type_test = lambda val: val
+
+        for path in path_list:
+            nl.depth = 0
+            val = _traverse_obj(obj, path)
+            if val is not None:
+                if nl.depth:
+                    for _ in range(nl.depth - 1):
+                        val = from_iterable(v for v in val if v is not None)
+                    val = [v for v in map(type_test, val) if v is not None]
+                    if val:
+                        return val if get_all else val[0]
+                else:
+                    val = type_test(val)
+                    if val is not None:
+                        return val
+        return default
+
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady|zive)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
-        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
+        'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
         'info_dict': {
-            'id': '61924494877246241',
+            'id': '61924494877028507',
             'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Život v Grónsku',
-            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
+            'title': 'Bonus 01 - En - Hyde Park Civilizace',
+            'description': 'English Subtittles',
             'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3350,
+            'duration': 81.3,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
+        # live stream
+        'url': 'http://www.ceskatelevize.cz/zive/ct1/',
         'info_dict': {
-            'id': '61924494877028507',
+            'id': '102',
             'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Bonus 01 - En',
-            'description': 'English Subtittles',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 81.3,
+            'title': r'ČT1 - živé vysílání online',
+            'description': 'Sledujte živé vysílání kanálu ČT1 online. Vybírat si můžete i z dalších kanálů České televize na kterémkoli z vašich zařízení.',
+            'is_live': True,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }, {
-        # live stream
+        # another
         'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
+        'only_matching': True,
         'info_dict': {
             'id': 402,
             'ext': 'mp4',
             'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'is_live': True,
         },
+        # 'skip': 'Georestricted to Czech Republic',
+    }, {
+        'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
+        'only_matching': True,
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Bogotart - Queer',
+            'description': 'Hlavní město Kolumbie v doprovodu queer umělců. Vroucí svět plný vášně, sebevědomí, ale i násilí a bolesti',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494877311053',
+                'ext': 'mp4',
+                'title': 'Bogotart - Queer (Varování 18+)',
+                'duration': 11.9,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Bogotart - Queer (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
         'params': {
             # m3u8 download
             'skip_download': True,
         },
-        'skip': 'Georestricted to Czech Republic',
     }, {
-        'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
+        # iframe embed
+        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
         'only_matching': True,
     }]
 
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
+        webpage, urlh = self._download_webpage_handle(url, playlist_id)
+        parsed_url = compat_urllib_parse_urlparse(urlh.geturl())
+        site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
+        playlist_title = self._og_search_title(webpage, default=None)
+        if site_name and playlist_title:
+            playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, 1)[0]
+        playlist_description = self._og_search_description(webpage, default=None)
+        if playlist_description:
+            playlist_description = playlist_description.replace('\xa0', ' ')
+
+        type_ = 'IDEC'
+        if re.search(r'(^/porady|/zive)/', parsed_url.path):
+            next_data = self._search_nextjs_data(webpage, playlist_id)
+            if '/zive/' in parsed_url.path:
+                idec = traverse_obj(next_data, ('props', 'pageProps', 'data', 'liveBroadcast', 'current', 'idec'), get_all=False)
+            else:
+                idec = traverse_obj(next_data, ('props', 'pageProps', 'data', ('show', 'mediaMeta'), 'idec'), get_all=False)
+                if not idec:
+                    idec = traverse_obj(next_data, ('props', 'pageProps', 'data', 'videobonusDetail', 'bonusId'), get_all=False)
+                    if idec:
+                        type_ = 'bonus'
+            if not idec:
+                raise ExtractorError('Failed to find IDEC id')
+            iframe_hash = self._download_webpage(
+                'https://www.ceskatelevize.cz/v-api/iframe-hash/',
+                playlist_id, note='Getting IFRAME hash')
+            query = {'hash': iframe_hash, 'origin': 'iVysilani', 'autoStart': 'true', type_: idec, }
+            webpage = self._download_webpage(
+                'https://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php',
+                playlist_id, note='Downloading player', query=query)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
-            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
+            self.raise_geo_restricted(NOT_AVAILABLE_STRING)
+        if any(not_found in webpage for not_found in ('Neplatný parametr pro videopřehrávač', 'IDEC nebyl nalezen', )):
+            raise ExtractorError('no video with IDEC available', video_id=idec, expected=True)
 
         type_ = None
         episode_id = None
@@ -100,7 +273,7 @@ def _real_extract(self, url):
         data = {
             'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
-            'requestUrl': compat_urllib_parse_urlparse(url).path,
+            'requestUrl': parsed_url.path,
             'requestSource': 'iVysilani',
         }
 
@@ -108,7 +281,7 @@ def _real_extract(self, url):
 
         for user_agent in (None, USER_AGENTS['Safari']):
             req = sanitized_Request(
-                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
@@ -130,9 +303,6 @@ def _real_extract(self, url):
             req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
             req.add_header('Referer', url)
 
-            playlist_title = self._og_search_title(webpage, default=None)
-            playlist_description = self._og_search_description(webpage, default=None)
-
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:
                 continue
@@ -167,7 +337,7 @@ def _real_extract(self, url):
                     entries[num]['formats'].extend(formats)
                     continue
 
-                item_id = item.get('id') or item['assetId']
+                item_id = str_or_none(item.get('id') or item['assetId'])
                 title = item['title']
 
                 duration = float_or_none(item.get('duration'))
@@ -181,8 +351,6 @@ def _real_extract(self, url):
 
                 if playlist_len == 1:
                     final_title = playlist_title or title
-                    if is_live:
-                        final_title = self._live_title(final_title)
                 else:
                     final_title = '%s (%s)' % (playlist_title, title)
 
@@ -200,6 +368,8 @@ def _real_extract(self, url):
         for e in entries:
             self._sort_formats(e['formats'])
 
+        if len(entries) == 1:
+            return entries[0]
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
     def _get_subtitles(self, episode_id, subs):
@@ -236,54 +406,3 @@ def _fix_subtitle(subtitle):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
-
-
-class CeskaTelevizePoradyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
-    _TESTS = [{
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # iframe embed
-        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        data_url = update_url_query(unescapeHTML(self._search_regex(
-            (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
-            webpage, 'iframe player url', group='url')), query={
-                'autoStart': 'true',
-        })
-
-        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..871f2a3d39c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -208,10 +208,7 @@
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import (
-    CeskaTelevizeIE,
-    CeskaTelevizePoradyIE,
-)
+from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE

From 01f720fe4a4dadfd1164a00086b0b19c84c99033 Mon Sep 17 00:00:00 2001
From: hatienl0i261299 <loiht.b17vt220@stu.ptit.edu.vn>
Date: Wed, 2 Mar 2022 19:43:44 +0700
Subject: [PATCH 146/394] Change name and update code follow convention

---
 youtube_dl/extractor/zingmp3.py | 70 +++++++++++++--------------------
 1 file changed, 27 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 75d64ee9a92..c1301a9dafe 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -1,23 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import hashlib
 import hmac
 import re
-from urllib.parse import urlencode
+from youtube_dl.compat import compat_urllib_parse_urlencode
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     try_get,
+    ExtractorError,
+    dict_get,
 )
 
 
 class ZingMp3BaseIE(InfoExtractor):
-    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>(?:%s))/[^/]+/(?P<id>\w+)(?:\.html|\?)'
+    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>(?:%s))/[^/]+/(?P<id>\w+)($|\W)'
     _GEO_COUNTRIES = ['VN']
-    _IS_UPDATE_COOKIES = False
     _DOMAIN = 'https://zingmp3.vn'
     _SLUG_API = {
         'bai-hat': '/api/v2/page/get/song',
@@ -29,21 +29,18 @@ class ZingMp3BaseIE(InfoExtractor):
         'song_streaming': '/api/v2/song/get/streaming',
     }
 
-    _TIMESTAMP = str(int(datetime.datetime.now().timestamp()))
     _API_KEY = '88265e23d4284f25963e6eedac8fbfa3'
     _SECRET_KEY = b'2aa2d1c561e809b267f3638c4a307aab'
 
     def _extract_item(self, item, song_id, type_url, fatal):
         item_id = item.get('encodeId') or song_id
-        title = item.get('title') or item.get('alias')
+        title = dict_get(item, ('title', 'alias')) or item['title']
 
         if type_url == 'video-clip':
             source = item.get('streaming')
         else:
-            api = self.get_api_with_signature(name_api=self._SLUG_API.get('song_streaming'), param={
-                'ctime': self._TIMESTAMP,
-                'id': item_id})
-            source = self.download_json(api, item_id).get('data')
+            api = self.get_api_with_signature(name_api=self._SLUG_API.get('song_streaming'), param={'id': item_id})
+            source = self._download_json(api, video_id=item_id).get('data')
 
         formats = []
         for k, v in (source or {}).items():
@@ -83,10 +80,8 @@ def _extract_item(self, item, song_id, type_url, fatal):
 
         lyric = item.get('lyric')
         if not lyric:
-            api = self.get_api_with_signature(name_api=self._SLUG_API.get("lyric"), param={
-                'ctime': self._TIMESTAMP,
-                'id': item_id})
-            info_lyric = self.download_json(api, item_id)
+            api = self.get_api_with_signature(name_api=self._SLUG_API.get("lyric"), param={'id': item_id})
+            info_lyric = self._download_json(api, video_id=item_id)
             lyric = try_get(info_lyric, lambda x: x['data']['file'])
         subtitles = {
             'origin': [{
@@ -100,13 +95,13 @@ def _extract_item(self, item, song_id, type_url, fatal):
             'id': item_id,
             'title': title,
             'formats': formats,
-            'thumbnail': item.get('thumbnail') or item.get('thumbnailM'),
+            'thumbnail': try_get(item, lambda x: x['thumbnail']) or try_get(item, lambda x: x['thumbnailM']),
             'subtitles': subtitles,
-            'duration': int_or_none(item.get('duration')),
+            'duration': int_or_none(try_get(item, lambda x: x['duration'])),
             'track': title,
-            'artist': item.get('artistsNames') or item.get('artists_names'),
-            'album': album.get('name') or album.get('title'),
-            'album_artist': album.get('artistsNames') or album.get('artists_names'),
+            'artist': try_get(item, lambda x: x['artistsNames']) or try_get(item, lambda x: x['artists_names']),
+            'album': try_get(album, lambda x: x['name']) or try_get(album, lambda x: x['title']),
+            'album_artist': try_get(album, lambda x: x['artistsNames']) or try_get(album, lambda x: x['artists_names']),
         }
 
     def _real_extract(self, url):
@@ -114,44 +109,33 @@ def _real_extract(self, url):
 
         song_id = mobj.group('id')
         type_url = mobj.group('type')
+        api = self.get_api_with_signature(name_api=self._SLUG_API[type_url], param={'id': song_id})
 
-        api = self.get_api_with_signature(name_api=self._SLUG_API[type_url], param={
-            'ctime': self._TIMESTAMP,
-            'id': song_id})
-        info = self.download_json(api, song_id)
-        return self._process_data(info.get('data'), song_id, type_url)
+        # Check the cookie file, if cookie file doesn't exist, create a dummy request to update the cookie
+        if not self._downloader.params.get('cookiefile'):
+            self._download_json(api, video_id=song_id, note='Dummy request to update cookie')
 
-    def download_json(self, api, video_id):
-        # For the first time making the request, we need to make one more temp request to update the cookie
-        # and the next time we don't need to do it.
-        # But if download with cookieFile, so we don't need to make one more temp request
-        if self._downloader.params.get('cookiefile'):
-            self._IS_UPDATE_COOKIES = True
+        info = self._download_json(api, video_id=song_id)
 
-        if not self._IS_UPDATE_COOKIES:
-            self._download_json(api, video_id=video_id, note='Update cookies for request')
-            self._IS_UPDATE_COOKIES = True
+        if not info:
+            raise ExtractorError('Can not extract data.')
 
-        return self._download_json(api, video_id=video_id)
+        return self._process_data(info.get('data'), song_id, type_url)
 
     def get_api_with_signature(self, name_api, param):
-        text = ''
-        for k, v in param.items():
-            text += f'{k}={v}'
+        sha256 = self.sha256_params(''.join('%s=%s' % (k, v) for k, v in param.items()))
 
-        sha256 = self.get_hash256(text)
         data = {
-            'ctime': param.get('ctime'),
             'apiKey': self._API_KEY,
-            'sig': self.get_hmac512(f'{name_api}{sha256}')
+            'sig': self.hmac512_string('%s%s' % (name_api, sha256))
         }
         data.update(param)
-        return f'{self._DOMAIN}{name_api}?{urlencode(data)}'
+        return '%s%s?%s' % (self._DOMAIN, name_api, compat_urllib_parse_urlencode(data))
 
-    def get_hash256(self, string):
+    def sha256_params(self, string):
         return hashlib.sha256(string.encode('utf-8')).hexdigest()
 
-    def get_hmac512(self, string):
+    def hmac512_string(self, string):
         return hmac.new(self._SECRET_KEY, string.encode('utf-8'), hashlib.sha512).hexdigest()
 
 

From a631e79b3344dd6dfbca3571940870be97cc6afc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 5 Mar 2022 13:58:02 +0000
Subject: [PATCH 147/394] [core] Implement a simple version  of yt-dlp's
 `--print` option

---
 README.md               |  9 +++++++++
 youtube_dl/YoutubeDL.py | 43 ++++++++++++++++++++++++++++++++---------
 youtube_dl/__init__.py  |  3 ++-
 youtube_dl/options.py   | 16 +++++++++++++++
 4 files changed, 61 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 2841ed68ff3..d8f8f21bad3 100644
--- a/README.md
+++ b/README.md
@@ -310,6 +310,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --get-filename                       Simulate, quiet but print output
                                          filename
     --get-format                         Simulate, quiet but print output format
+    -O, --print TEMPLATE                 Simulate, quiet but print the given fields.
+                                         Either a field name or similar formatting
+                                         as the output template can be used
     -j, --dump-json                      Simulate, quiet but print JSON
                                          information. See the "OUTPUT TEMPLATE"
                                          for a description of available keys.
@@ -620,6 +623,12 @@ Available for the media that is a track or a part of a music album:
  - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to
  - `release_year` (numeric): Year (YYYY) when the album was released
 
+Available only when used in `--print`:
+
+ - `urls` (string): The URLs of all requested formats, one in each line
+ - `duration_string` (string): Length of the video (HH:mm:ss)
+ - `filename` (string): Name of the video file. Note that the actual filename may be different due to post-processing. Use `--exec echo` to get the name after all postprocessing is complete
+
 Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default).
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj`, this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 69736acffa6..596637be6af 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1744,25 +1744,49 @@ def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
         return subs
 
     def __forced_printings(self, info_dict, filename, incomplete):
+        FIELD_ALIASES = {}
+
         def print_mandatory(field):
+            actual_field = FIELD_ALIASES.get(field, field)
             if (self.params.get('force%s' % field, False)
-                    and (not incomplete or info_dict.get(field) is not None)):
-                self.to_stdout(info_dict[field])
+                    and (not incomplete or info_dict.get(actual_field) is not None)):
+                self.to_stdout(info_dict[actual_field])
 
         def print_optional(field):
             if (self.params.get('force%s' % field, False)
                     and info_dict.get(field) is not None):
                 self.to_stdout(info_dict[field])
 
+        info_dict = info_dict.copy()
+        info_dict['duration_string'] = (  # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
+            formatSeconds(info_dict['duration'])
+            if info_dict.get('duration', None) is not None
+            else None)
+        if info_dict.get('resolution') is None:
+            info_dict['resolution'] = self.format_resolution(info_dict, default=None)
+        if filename is not None:
+            info_dict['filename'] = filename
+        if info_dict.get('requested_formats') is not None:
+            # For RTMP URLs, also include the playpath
+            info_dict['urls'] = '\n'.join(f['url'] + f.get('play_path', '') for f in info_dict['requested_formats'])
+        elif 'url' in info_dict:
+            info_dict['urls'] = info_dict['url'] + info_dict.get('play_path', '')
+        if 'urls' in info_dict:
+            FIELD_ALIASES['url'] = 'urls'
+
+        for tmpl in self.params.get('forceprint', []):
+            if re.match(r'\w+$', tmpl):
+                tmpl = '%({0})s'.format(tmpl)
+            try:
+                out_txt = tmpl % info_dict
+            except KeyError:
+                self.report_warning('Skipping invalid print string "%s"' % (tmpl, ))
+                continue
+            self.to_stdout(out_txt)
+
         print_mandatory('title')
         print_mandatory('id')
-        if self.params.get('forceurl', False) and not incomplete:
-            if info_dict.get('requested_formats') is not None:
-                for f in info_dict['requested_formats']:
-                    self.to_stdout(f['url'] + f.get('play_path', ''))
-            else:
-                # For RTMP URLs, also include the playpath
-                self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
+        print_mandatory('url')
         print_optional('thumbnail')
         print_optional('description')
         if self.params.get('forcefilename', False) and filename is not None:
@@ -1770,6 +1794,7 @@ def print_optional(field):
         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
             self.to_stdout(formatSeconds(info_dict['duration']))
         print_mandatory('format')
+
         if self.params.get('forcejson', False):
             self.to_stdout(json.dumps(info_dict))
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e1bd67919cf..4a44752cbf3 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -243,7 +243,7 @@ def parse_retries(retries):
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      ' template'.format(outtmpl))
 
-    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_getting = opts.print_ or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     any_printing = opts.print_json
     download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
@@ -333,6 +333,7 @@ def parse_retries(retries):
         'forceduration': opts.getduration,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
+        'forceprint': opts.print_,
         'forcejson': opts.dumpjson or opts.print_json,
         'dump_single_json': opts.dump_single_json,
         'simulate': opts.simulate or any_getting,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0a0641bd481..0a262729fab 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -12,6 +12,7 @@
     compat_getenv,
     compat_kwargs,
     compat_shlex_split,
+    compat_str,
 )
 from .utils import (
     preferredencoding,
@@ -111,6 +112,14 @@ def _format_option_string(option):
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
 
+    def _list_from_options_callback(option, opt_str, value, parser, append=True, delim=',', process=compat_str.strip):
+        # append can be True, False or -1 (prepend)
+        current = list(getattr(parser.values, option.dest)) if append else []
+        value = list(filter(None, [process(value)] if delim is None else map(process, value.split(delim))))
+        setattr(
+            parser.values, option.dest,
+            current + value if append is True else value + current)
+
     # No need to wrap help messages if we're on a wide console
     columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80
@@ -590,6 +599,13 @@ def _comma_separated_values_options_callback(option, opt_str, value, parser):
         '--skip-download',
         action='store_true', dest='skip_download', default=False,
         help='Do not download the video')
+    verbosity.add_option(
+        '-O', '--print', metavar='TEMPLATE',
+        action='callback', dest='print_', type='str', default=[],
+        callback=_list_from_options_callback, callback_kwargs={'delim': None},
+        help=(
+            'Simulate, quiet but print the given fields. Either a field name '
+            'or similar formatting as the output template can be used'))
     verbosity.add_option(
         '-g', '--get-url',
         action='store_true', dest='geturl', default=False,

From 34e1010545c5aba424cb77cf0b4cbfc0f3830408 Mon Sep 17 00:00:00 2001
From: DarkFighterLuke <luca-consoli@live.it>
Date: Thu, 10 Mar 2022 15:07:24 +0100
Subject: [PATCH 148/394] Fix view_count

---
 youtube_dl/extractor/pornhub.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index e2e1500ff6e..3f5ee40cdfc 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -20,8 +20,8 @@
     merge_dicts,
     NO_DEFAULT,
     orderedSet,
+    parse_count,
     remove_quotes,
-    str_to_int,
     update_url_query,
     urlencode_postdata,
     url_or_none,
@@ -132,7 +132,7 @@ class PornHubIE(PornHubBaseIE):
                     ''' % PornHubBaseIE._PORNHUB_HOST_RE
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
-        'md5': 'a6391306d050e4547f62b3f485dd9ba9',
+        'md5': 'fc5caada049bee2e944229ad850ef5db',
         'info_dict': {
             'id': '648719015',
             'ext': 'mp4',
@@ -255,7 +255,7 @@ def _extract_urls(webpage):
             webpage)
 
     def _extract_count(self, pattern, webpage, name):
-        return str_to_int(self._search_regex(
+        return parse_count(self._search_regex(
             pattern, webpage, '%s count' % name, fatal=False))
 
     def _real_extract(self, url):
@@ -466,7 +466,7 @@ def extract_vote_count(kind, name):
                 webpage, name)
 
         view_count = self._extract_count(
-            r'<span class="count">([\d,\.]+)</span> [Vv]iews', webpage, 'view')
+            r'<span class="count">(?P<views>.+?)</span> [Vv]iews', webpage, 'view')
         like_count = extract_vote_count('Up', 'like')
         dislike_count = extract_vote_count('Down', 'dislike')
         comment_count = self._extract_count(

From 8e3afbc51b691ca3796f652e7e116df5b9167334 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 13 Mar 2022 16:22:34 +0100
Subject: [PATCH 149/394] cleaning

---
 README.md              | 7 ++++---
 docs/supportedsites.md | 3 +--
 youtube_dl/version.py  | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index d8f8f21bad3..68ce7f6d897 100644
--- a/README.md
+++ b/README.md
@@ -300,6 +300,10 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -s, --simulate                       Do not download the video and do not
                                          write anything to disk
     --skip-download                      Do not download the video
+    -O, --print TEMPLATE                 Simulate, quiet but print the given
+                                         fields. Either a field name or similar
+                                         formatting as the output template can
+                                         be used
     -g, --get-url                        Simulate, quiet but print URL
     -e, --get-title                      Simulate, quiet but print title
     --get-id                             Simulate, quiet but print id
@@ -310,9 +314,6 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --get-filename                       Simulate, quiet but print output
                                          filename
     --get-format                         Simulate, quiet but print output format
-    -O, --print TEMPLATE                 Simulate, quiet but print the given fields.
-                                         Either a field name or similar formatting
-                                         as the output template can be used
     -j, --dump-json                      Simulate, quiet but print JSON
                                          information. See the "OUTPUT TEMPLATE"
                                          for a description of available keys.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ba242f9d14d..a9314abe9d7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -170,7 +170,6 @@
  - **CCTV**: 央视网
  - **CDA**
  - **CeskaTelevize**
- - **CeskaTelevizePorady**
  - **channel9**: Channel 9
  - **CharlieRose**
  - **Chaturbate**
@@ -1235,7 +1234,7 @@
  - **ZDF**
  - **ZDFChannel**
  - **Zhihu**
- - **zingmp3**: mp3.zing.vn
+ - **zingmp3**: zingmp3.vn
  - **zingmp3:album**
  - **zoom**
  - **Zype**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4b4389cc404..ad32023dbf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.28'
+__version__ = '2021.03.13'

From 9a6b3d9dac63d7fea68949658e16db2786125499 Mon Sep 17 00:00:00 2001
From: Martin Vician <info@vician.net>
Date: Wed, 16 Mar 2022 17:44:10 +0000
Subject: [PATCH 150/394] Added mujrozhlas.cz extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/mujrozhlas.py | 53 ++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/mujrozhlas.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..daad998922e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -718,6 +718,7 @@
     MTVJapanIE,
 )
 from .muenchentv import MuenchenTVIE
+from .mujrozhlas import MujRozhlasIE
 from .mwave import MwaveIE, MwaveMeetGreetIE
 from .mychannels import MyChannelsIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
diff --git a/youtube_dl/extractor/mujrozhlas.py b/youtube_dl/extractor/mujrozhlas.py
new file mode 100644
index 00000000000..ba2f1eea473
--- /dev/null
+++ b/youtube_dl/extractor/mujrozhlas.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_start,
+    clean_html,
+)
+
+
+class MujRozhlasIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mujrozhlas\.cz/(?P<id>[a-zA-Z0-9\-\/]+)'
+    _TESTS = [{
+        'url': 'https://www.mujrozhlas.cz/meteor/meteor-o-nejvetsim-matematikovi-nekonecnem-vesmiru-skakajicim-pavoukovi-hrani-surikat',
+        'info_dict': {
+            'id': 'meteor/meteor-o-nejvetsim-matematikovi-nekonecnem-vesmiru-skakajicim-pavoukovi-hrani-surikat',
+            'ext': 'mp3',
+            'title': 'Meteor o největším matematikovi, nekonečném vesmíru, skákajícím pavoukovi a hraní surikat',
+            'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka'
+        }
+    },{
+        'url': 'https://www.mujrozhlas.cz/podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
+        'info_dict': {
+            'id': 'podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
+            'ext': 'mp3',
+            'title': 'Meteor o největším matematikovi, nekonečném vesmíru, skákajícím pavoukovi a hraní surikat',
+            'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka'
+        }
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        content_id = self._html_search_regex(r'\"contentId\":\"(.+?)\"',webpage,'content_id',default=None)
+        content_url = 'https://api.mujrozhlas.cz/episodes/' + content_id
+
+        content = self._download_json(content_url, content_id)
+        audio_url = content['data']['attributes']['audioLinks'][0]['url']
+        duration = content['data']['attributes']['audioLinks'][0]['duration']
+        title = content['data']['attributes']['title']
+        description = clean_html(content['data']['attributes']['description'])
+
+        return {
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+            'description': description,
+            'duration': int(duration),
+            'vcodec': 'none',
+        }

From fb13bb6598921e5bfa31d933960d70afbd2b4014 Mon Sep 17 00:00:00 2001
From: Martin Vician <info@vician.net>
Date: Wed, 16 Mar 2022 17:51:50 +0000
Subject: [PATCH 151/394] Updated to flake8

---
 youtube_dl/extractor/mujrozhlas.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mujrozhlas.py b/youtube_dl/extractor/mujrozhlas.py
index ba2f1eea473..2e494566b0e 100644
--- a/youtube_dl/extractor/mujrozhlas.py
+++ b/youtube_dl/extractor/mujrozhlas.py
@@ -4,7 +4,6 @@
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    remove_start,
     clean_html,
 )
 
@@ -19,7 +18,7 @@ class MujRozhlasIE(InfoExtractor):
             'title': 'Meteor o největším matematikovi, nekonečném vesmíru, skákajícím pavoukovi a hraní surikat',
             'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka'
         }
-    },{
+    }, {
         'url': 'https://www.mujrozhlas.cz/podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
         'info_dict': {
             'id': 'podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
@@ -34,7 +33,7 @@ def _real_extract(self, url):
 
         webpage = self._download_webpage(url, audio_id)
 
-        content_id = self._html_search_regex(r'\"contentId\":\"(.+?)\"',webpage,'content_id',default=None)
+        content_id = self._html_search_regex(r'\"contentId\":\"(.+?)\"', webpage, 'content_id', default=None)
         content_url = 'https://api.mujrozhlas.cz/episodes/' + content_id
 
         content = self._download_json(content_url, content_id)
@@ -48,6 +47,6 @@ def _real_extract(self, url):
             'url': audio_url,
             'title': title,
             'description': description,
-            'duration': int(duration),
+            'duration': int_or_none(duration),
             'vcodec': 'none',
         }

From 369a15544a25f6d43016721a1c7b814cca10abeb Mon Sep 17 00:00:00 2001
From: lihan7 <lihan7@xiaomi.com>
Date: Fri, 25 Mar 2022 15:46:28 +0800
Subject: [PATCH 152/394] [extractor/bilibili] Fix path "/audio/auxxxxx"
 download return 403

---
 youtube_dl/extractor/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index bff6ea194d7..d42f0e98a4b 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -369,6 +369,11 @@ def _real_extract(self, url):
             'filesize': int_or_none(play_data.get('size')),
         }]
 
+        for a_format in formats:
+            a_format.setdefault('http_headers', {}).update({
+                'Referer': url,
+            })
+
         song = self._call_api('song/info', au_id)
         title = song['title']
         statistic = song.get('statistic') or {}

From 3ec2a3d4ead26e014f4d187e79bd53cb6c894104 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 25 Mar 2022 13:04:58 +0000
Subject: [PATCH 153/394] [XVideos] Support playlists, searches and channels

---
 youtube_dl/extractor/extractors.py |   8 +-
 youtube_dl/extractor/xvideos.py    | 257 ++++++++++++++++++++++++++++-
 2 files changed, 257 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..b8fd501231a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1579,7 +1579,13 @@
 from .xstream import XstreamIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
-from .xvideos import XVideosIE
+from .xvideos import (
+    XVideosIE,
+    XVideosChannelIE,
+    XVideosPlaylistIE,
+    XVideosRelatedIE,
+    XVideosSearchIE,
+)
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index e63d4690d4b..1b2c0a5bc04 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -1,15 +1,26 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_parse_qs,
+    compat_str,
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
+)
 from ..utils import (
     clean_html,
     determine_ext,
+    extract_attributes,
     ExtractorError,
     int_or_none,
     parse_duration,
+    try_get,
+    url_basename,
+    urljoin,
 )
 
 
@@ -17,13 +28,13 @@ class XVideosIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[^/]+\.)?xvideos2?\.com/video|
+                            (?:[^/]+\.)?xvideos2?\.com/(?:video|prof-video-click/model/[^/]+/)|
                             (?:www\.)?xvideos\.es/video|
-                            flashservice\.xvideos\.com/embedframe/|
+                            (?:www|flashservice)\.xvideos\.com/embedframe/|
                             static-hw\.xvideos\.com/swf/xv-player\.swf\?.*?\bid_video=
                         )
-                        (?P<id>[0-9]+)
-                    '''
+                        (?P<id>\d+)
+                 '''
     _TESTS = [{
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
         'md5': '14cea69fcb84db54293b1e971466c2e1',
@@ -34,6 +45,18 @@ class XVideosIE(InfoExtractor):
             'duration': 108,
             'age_limit': 18,
         }
+    }, {
+        # Broken HLS formats
+        'url': 'https://www.xvideos.com/video65982001/what_s_her_name',
+        'md5': 'b82d7d7ef7d65a84b1fa6965f81f95a5',
+        'info_dict': {
+            'id': '65982001',
+            'ext': 'mp4',
+            'title': 'what\'s her name?',
+            'duration': 120,
+            'age_limit': 18,
+            'thumbnail': r're:^https://img-hw.xvideos-cdn.com/.+\.jpg',
+        }
     }, {
         'url': 'https://flashservice.xvideos.com/embedframe/4588838',
         'only_matching': True,
@@ -78,6 +101,12 @@ class XVideosIE(InfoExtractor):
         'only_matching': True
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        EXCLUDE_IE = (XVideosRelatedIE, )
+        return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
+                else super(XVideosIE, cls).suitable(url))
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -125,9 +154,11 @@ def _real_extract(self, url):
                 r'setVideo([^(]+)\((["\'])(http.+?)\2\)', webpage):
             format_id = kind.lower()
             if format_id == 'hls':
-                formats.extend(self._extract_m3u8_formats(
+                hls_formats = self._extract_m3u8_formats(
                     format_url, video_id, 'mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+                self._check_formats(hls_formats, video_id)
+                formats.extend(hls_formats)
             elif format_id in ('urllow', 'urlhigh'):
                 formats.append({
                     'url': format_url,
@@ -145,3 +176,215 @@ def _real_extract(self, url):
             'thumbnails': thumbnails,
             'age_limit': 18,
         }
+
+
+class XVideosPlaylistIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:[^/]+\.)?xvideos2?\.com/
+                          (?:c(?:/[sm]:[^/]+)*|
+                             profiles|
+                             favorite)/
+                            (?P<id>[^#?/]+)
+                  '''
+    _TESTS = []
+
+    def _extract_videos_from_json_list(self, json_list, path='video'):
+        return (
+            'https://www.xvideos.com/%s%d' % (path, x.get('id'), )
+            for x in json_list if isinstance(x, dict))
+
+    def _get_playlist_url(self, url, playlist_id):
+        """URL of first playlist page"""
+        id_match = re.match(self._VALID_URL, url).groupdict()
+        video_sort = id_match.get('sort')
+        if video_sort:
+            url, _ = compat_urlparse.urldefrag(url)
+            if url.endswith('/'):
+                url = url[:-1]
+            url = '%s/%s' % (url, video_sort.replace('-', '/'))
+        return url
+
+    def _get_next_page(self, url, num, page):
+        '''URL of num'th continuation page of url'''
+        if page.startswith('{'):
+            url, sub = re.subn(r'(/)(\d{1,7})($|[#?/])', r'\g<1>%d\3' % (num, ), url)
+            if sub == 0:
+                url += '/%d' % (num, )
+            return url
+        next_page = self._search_regex(
+            r'''(?s)(<a\s[^>]*?\bclass\s*=\s*(?P<q>'|").*?\bnext-page\b.*?(?P=q)[^>]*?>)''',
+            page, 'next page', default=None)
+        if next_page:
+            next_page = extract_attributes(next_page)
+            next_page = next_page.get('href')
+            if next_page:
+                return urljoin(url, next_page)
+        return False
+
+    def _extract_videos(self, url, playlist_id, num, page):
+        """Get iterable videos plus stop flag"""
+        return ((
+            'https://www.xvideos.com/video' + x.group('video_id')
+            for x in re.finditer(r'''<div\s[^>]*?id\s*=\s*(\'|")video_(?P<video_id>[0-9]+)\1''', page)),
+            None)
+
+    def _real_extract(self, url):
+        id_match = re.match(self._VALID_URL, url).groupdict()
+
+        playlist_id = id_match['id']
+        if url.endswith(playlist_id):
+            url += '/0'
+        next_page = self._get_playlist_url(url, playlist_id)
+        matches = []
+        for count in itertools.count(0):
+            webpage = self._download_webpage(
+                next_page,
+                '%s (+%d)' % (playlist_id, count) if count > 0 else playlist_id)
+
+            vids, stop = self._extract_videos(next_page, playlist_id, count, webpage)
+
+            if vids:
+                matches.append(vids)
+
+            if stop:
+                break
+            next_page = self._get_next_page(next_page, count + 1, webpage)
+            if not next_page:
+                break
+
+        return self.playlist_from_matches(
+            itertools.chain.from_iterable(matches), playlist_id)
+
+
+class XVideosRelatedIE(XVideosPlaylistIE):
+    _VALID_URL = XVideosIE._VALID_URL + r'(?:/[^/]+)*?\#_related-(?P<related>videos|playlists)'
+
+    _TESTS = []
+
+    def _extract_videos(self, url, playlist_id, num, page):
+        id_match = re.match(self._VALID_URL, url).groupdict()
+        related = id_match.get('related')
+        if not related:
+            return super(XVideosRelatedIE, self)._extract_videos(url, playlist_id, num, page)
+
+        if related == 'videos':
+            related_json = self._search_regex(
+                r'(?s)videos_related\s*=\s*(\[.*?])\s*;',
+                page, 'related', default='[]')
+            related_json = self._parse_json(related_json, playlist_id, fatal=False) or []
+            return (self._extract_videos_from_json_list(related_json), True)
+        # playlists
+        related_json = self._download_json(
+            'https://www.xvideos.com/video-playlists/' + playlist_id, playlist_id, fatal=False)
+        return (
+            self._extract_videos_from_json_list(
+                try_get(related_json, lambda x: x['playlists'], list) or [],
+                path='favorite/'),
+            True)
+
+
+class XVideosChannelIE(XVideosPlaylistIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:[^/]+\.)?xvideos2?\.com/
+                          (?:
+                             (?:amateur-|pornstar-|model-)?channel|
+                             pornstar
+                          )s/
+                            (?P<id>[^#?/]+)
+                              (?:\#_tab(?P<tab>Videos|Favorites|Playlists|AboutMe)(?:,(?P<sort>[^,]+))?)?
+                 '''
+    _TESTS = [{
+        'url': 'https://www.xvideos.com/pornstar-channels/sienna-west',
+        'playlist_mincount': 5,
+    }, ]
+
+    def _get_playlist_url(self, url, playlist_id):
+        webpage = self._download_webpage(url, playlist_id)
+        id_match = re.match(self._VALID_URL, url).groupdict()
+        tab = (id_match.get('tab') or '').lower()
+        if tab:
+            if tab in ('videos', 'favorites'):
+                url, frag = compat_urlparse.urldefrag(url)
+                if not url.endswith('/'):
+                    url += '/'
+                frag = frag.split(',')
+                url += tab
+                if tab == 'videos':
+                    url += '/' + (frag[1] if len(frag) > 1 else 'best')
+                url += '/0'
+            return url
+
+        # activity
+        conf = self._search_regex(
+            r'(?s)\.\s*xv\s*\.\s*conf\s*=\s*(\{.*?})[\s;]*</script',
+            webpage, 'XV conf')
+        conf = self._parse_json(conf, playlist_id)
+        act = try_get(conf,
+                      ((lambda x: x['dyn'][y])
+                       for y in ('page_main_cat', 'user_main_cat')),
+                      compat_str) or 'straight'
+
+        url, _ = compat_urlparse.urldefrag(url)
+        if url.endswith('/'):
+            url = url[:-1]
+
+        return '%s/activity/%s' % (url, act, )
+
+    def _get_next_page(self, url, num, page):
+        if page.startswith('{') or '#_tab' in url:
+            return super(XVideosChannelIE, self)._get_next_page(url, num, page)
+
+        act_time = int_or_none(url_basename(url)) or 0
+        last_act = int(self._search_regex(
+            r'(?s)id\s*=\s*"?activity-event-(\d{10})(?!.*id\s*=\s*"?activity-event-\d+.*).+$',
+            page, 'last activity', default=act_time))
+        if last_act == act_time:
+            return False
+        return (
+            url.replace('/%d' % (act_time, ), '/%d' % (last_act, ))
+            if act_time
+            else url + ('/%d' % (last_act, )))
+
+    def _extract_videos(self, url, playlist_id, num, page):
+        tab = next((x for x in ('videos', 'favorites') if '/%s/' % (x, ) in url), None)
+        if tab == 'videos':
+            tab_json = self._parse_json(page, playlist_id, fatal=False) or {}
+            more = try_get(tab_json, lambda x: x['current_page'] + 1, int)
+            more = int_or_none(more, scale=tab_json.get('nb_videos'), invscale=tab_json.get('nb_per_page'), default=0)
+            return (
+                self._extract_videos_from_json_list(
+                    try_get(tab_json, lambda x: x['videos'], list) or []),
+                more > 0)
+
+        if tab == 'favorites':
+            return ((
+                'https://www.xvideos.com' + x.group('playlist')
+                for x in re.finditer(r'''<a\s[^>]*?href\s*=\s*('|")(?P<playlist>/favorite/\d+/[^#?]+?)\1''', page)),
+                None)
+
+        return super(XVideosChannelIE, self)._extract_videos(url, playlist_id, num, page)
+
+
+class XVideosSearchIE(XVideosPlaylistIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:[^/]+\.)?xvideos2?\.com/
+                          \?k=(?P<id>[^#?/&]+)
+                 '''
+    _TESTS = [{
+        # uninteresting search with probably at least two pages of results,
+        # but not too many more
+        'url': 'http://www.xvideos.com/?k=libya&sort=length',
+        'playlist_mincount': 30,
+    }, ]
+
+    def _get_next_page(self, url, num, page):
+        parsed_url = compat_urlparse.urlparse(url)
+        qs = compat_parse_qs(parsed_url.query)
+        qs['p'] = [num]
+        parsed_url = (
+            list(parsed_url[:4])
+            + [compat_urllib_parse_urlencode(qs, True), None])
+        return compat_urlparse.urlunparse(parsed_url), False

From d4ac3ac47f25248fa463783dccc30f12c529cd04 Mon Sep 17 00:00:00 2001
From: Martin Vician <info@vician.net>
Date: Fri, 25 Mar 2022 16:18:21 +0000
Subject: [PATCH 154/394] No default for content_id, better content parsing

---
 youtube_dl/extractor/mujrozhlas.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mujrozhlas.py b/youtube_dl/extractor/mujrozhlas.py
index 2e494566b0e..a28559d4a72 100644
--- a/youtube_dl/extractor/mujrozhlas.py
+++ b/youtube_dl/extractor/mujrozhlas.py
@@ -9,7 +9,7 @@
 
 
 class MujRozhlasIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mujrozhlas\.cz/(?P<id>[a-zA-Z0-9\-\/]+)'
+    _VALID_URL = r'https?://(?:www\.)?mujrozhlas\.cz/(?P<id>[a-zA-Z0-9/-]+)'
     _TESTS = [{
         'url': 'https://www.mujrozhlas.cz/meteor/meteor-o-nejvetsim-matematikovi-nekonecnem-vesmiru-skakajicim-pavoukovi-hrani-surikat',
         'info_dict': {
@@ -33,14 +33,16 @@ def _real_extract(self, url):
 
         webpage = self._download_webpage(url, audio_id)
 
-        content_id = self._html_search_regex(r'\"contentId\":\"(.+?)\"', webpage, 'content_id', default=None)
+        content_id = self._html_search_regex(r'\"contentId\":\"(.+?)\"', webpage, 'content_id')
         content_url = 'https://api.mujrozhlas.cz/episodes/' + content_id
 
         content = self._download_json(content_url, content_id)
-        audio_url = content['data']['attributes']['audioLinks'][0]['url']
-        duration = content['data']['attributes']['audioLinks'][0]['duration']
-        title = content['data']['attributes']['title']
-        description = clean_html(content['data']['attributes']['description'])
+        attrs = content['data']['attributes']
+        title = attrs['title']
+        audio_info = content['data']['attributes']['audioLinks'][0]
+        duration = audio_info.get('duration')
+        description = clean_html(attrs.get('description'))
+        audio_url = audio_info.get('url')
 
         return {
             'id': audio_id,

From 2b8eac2d3ab49111271a7b38efad145cdc2a4521 Mon Sep 17 00:00:00 2001
From: alyssadev <alyssa.dev.smith@gmail.com>
Date: Tue, 29 Mar 2022 18:17:31 +1000
Subject: [PATCH 155/394] [nbl] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nbl.py        | 69 ++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/nbl.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..e48f67424de 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -749,6 +749,7 @@
     NBCSportsStreamIE,
     NBCSportsVPlayerIE,
 )
+from .nbl import NBLIE
 from .ndr import (
     NDRIE,
     NJoyIE,
diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py
new file mode 100644
index 00000000000..0c3c1e15932
--- /dev/null
+++ b/youtube_dl/extractor/nbl.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class NBLIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P<id>[0-9]+)'  # /(?P<display_id>.+)'
+    _TESTS = [
+        {
+            'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights',
+            'md5': '684a1126879371c3137850b8474ae3c8',
+            'info_dict': {
+                'id': '1310086',
+                'ext': 'mp4',
+                'title': 'Perth Wildcats vs. Sydney Kings - Game Highlights',
+                'description': 'Watch the Game Highlights from Perth Wildcats vs. Sydney Kings, 03/26/2022',
+                'upload_date': '20220326'
+            }
+        },
+        {
+            'url': 'https://nbl.com.au/tv/condensed-games/1310087/Perth-Wildcats-vs.-Sydney-Kings---Condensed-Game',
+            'md5': '1f9ac9ea04dc4024e50b27593860a782',
+            'info_dict': {
+                'id': '1310087',
+                'ext': 'mp4',
+                'title': 'Perth Wildcats vs. Sydney Kings - Condensed Game',
+                'description': 'Watch the Condensed Game from Perth Wildcats vs. Sydney Kings, 03/26/2022',
+                'upload_date': '20220326'
+            }
+        },
+        {
+            'url': 'https://nbl.com.au/tv/game-replays/1303323/NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings',
+            'md5': '8505f02156f756e865a1aa80050eb768',
+            'expected_warnings': ['unable to extract OpenGraph description.+'],
+            'info_dict': {
+                'id': '1303323',
+                'ext': 'mp4',
+                'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings',
+                'upload_date': '20220326'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
+        stream_url = self._download_json(
+            "https://ott.nbl.com.au/api/v2/content/" + video_id + "/access/hls",
+            video_id,
+            data=b"",
+            headers={
+                "Referer": "https://ott.nbl.com.au/en-int/embed/" + video_id,
+                "Origin": "https://ott.nbl.com.au",
+                "Content-Length": "0"
+            }
+        )["data"]["stream"]
+        formats = self._extract_m3u8_formats(stream_url, video_id, ext="mp4")
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._og_search_description(webpage, default=None),
+            'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date')),
+            'formats': formats
+        }

From 0d1bb1ce255f67d18ef4e53ab1d88303c0a71a91 Mon Sep 17 00:00:00 2001
From: alyssadev <alyssa.dev.smith@gmail.com>
Date: Tue, 29 Mar 2022 18:29:04 +1000
Subject: [PATCH 156/394] [nbl] re-enable display_id match for testing

---
 youtube_dl/extractor/nbl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py
index 0c3c1e15932..da6cf492d8e 100644
--- a/youtube_dl/extractor/nbl.py
+++ b/youtube_dl/extractor/nbl.py
@@ -6,7 +6,7 @@
 
 
 class NBLIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P<id>[0-9]+)'  # /(?P<display_id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P<id>[0-9]+)/(?P<display_id>.+)'
     _TESTS = [
         {
             'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights',

From 3033db0217e58b4c264683a49dc71660efea317a Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Wed, 30 Mar 2022 17:05:20 +0100
Subject: [PATCH 157/394] Fix getting HLS media, thumbnail

---
 youtube_dl/extractor/beeg.py | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 551b789e557..249e61c86c2 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,14 +1,20 @@
+# coding: utf-8
+
 from __future__ import unicode_literals
 
+import itertools
+
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urlparse,
+    compat_zip as zip,
 )
 from ..utils import (
     dict_get,
     int_or_none,
     parse_iso8601,
+    str_or_none,
     try_get,
     unified_timestamp,
     urljoin,
@@ -51,7 +57,7 @@ class BeegIE(InfoExtractor):
         'url': 'https://beeg.com/-01882333214',
         'md5': 'f8fd97a58a09bc6d2ac25b6bbc98c220',
         'info_dict': {
-            'id': '01882333214',
+            'id': '1882333214',
             'ext': 'mp4',
             'title': 'Be mine',
             'description': 'Lil cam dork living in Canada with 4 kitties',
@@ -60,6 +66,11 @@ class BeegIE(InfoExtractor):
             'duration': 559,
             'tags': list,
             'age_limit': 18,
+            'thumbnail': r're:https?://thumbs\.beeg\.com/videos/1882333214/\d+\.jpe?g',
+        },
+        'params': {
+            # ignore variable HLS file data
+            'skip_download': True,
         },
     }]
 
@@ -83,7 +94,7 @@ def _old_real_extract(self, url, video_id, webpage):
             query = {'v': 1}
 
         def reverse_enumerate(l):
-            return zip(reversed(zip(*enumerate(l))[0]), l)
+            return zip(reversed(next(zip(*enumerate(l)), [])), l)
 
         for rcnt, api_path in reverse_enumerate(('', 'api.')):
             video = self._download_json(
@@ -147,7 +158,9 @@ def _real_extract(self, url):
             video_data = try_get(video, lambda x: x['file'], dict) or {}
             video_data.update(video_data.get('stuff', {}))
             video_data.update(try_get(video, lambda x: x['fc_facts'][0], dict))
-            for format_id, key in try_get(video_data, lambda x: iter(x['resources'].items())) or []:
+            for format_id, key in itertools.chain(
+                    try_get(video_data, lambda x: x['resources'].items()) or [],
+                    try_get(video_data, lambda x: x['hls_resources'].items()) or []):
                 if not format_id:
                     continue
                 formats.append({
@@ -157,24 +170,25 @@ def _real_extract(self, url):
                 })
             if formats:
                 self._sort_formats(formats)
-                title = video_data.get('sf_name') or 'Beeg video %s' % video_id
-                tags = try_get(video_data, lambda v: [tag['tg_slug'] for tag in v['tags'] if tag.get('tg_slug')])
+                tags = try_get(video, lambda v: [tag['tg_slug'] for tag in v['tags'] if tag.get('tg_slug')])
                 result = {
-                    'id': video_id,
+                    'id': str_or_none(video.get('fc_file_id')) or video_id,
                     'title': video_data.get('sf_name') or 'Beeg video %s' % video_id,
-                    'description': video_data.get('sf_story', title),
+                    'description': video_data.get('sf_story'),
                     'timestamp': parse_iso8601(video_data.get('fc_created')),
                     'duration': int_or_none(dict_get(video_data, ('fl_duration', 'sf_duration'))),
                     'tags': tags,
                     'formats': formats,
-                    'age_limit': self._rta_search(webpage),
+                    'age_limit': self._rta_search(webpage) or 18,
                     'height': int_or_none(video_data.get('fl_height')),
                     'width': int_or_none(video_data.get('fl_width')),
                 }
-                set_id = video_data['resources'].get('set_id')
-                thumb_id = try_get(video_data, lambda x: x['fc_facts'][0]['fc_thumbs'][0], int)
-                if set_id and thumb_id:
+                thumb_id = try_get(video_data, lambda x: x['fc_thumbs'][0], int)
+                if thumb_id is not None:
+                    set_id = video_data.get('resources', {}).get('set_id')
                     result['thumbnail'] = (
+                        'https://thumbs.beeg.com/videos/%s/%d.jpg' % (result['id'], thumb_id)
+                        if set_id is None else
                         'https://thumbs-015.externulls.com/sets/%s/thumbs/%s-%0.4d.jpg' % (set_id, set_id, thumb_id))
                 return result
 

From aded139c4f9d9adb7a992fa24d87be8ef7fbe1f5 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Thu, 31 Mar 2022 03:56:02 +0500
Subject: [PATCH 158/394] adding https://commons.wikimedia.org to youtube-dl

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/wikimedia.py  | 35 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/wikimedia.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..a6391997ed9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1659,3 +1659,4 @@
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE
+from .wikimedia import WikimediaIE
diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
new file mode 100644
index 00000000000..fec867a122c
--- /dev/null
+++ b/youtube_dl/extractor/wikimedia.py
@@ -0,0 +1,35 @@
+from .common import InfoExtractor
+import re
+import requests
+import urllib.parse
+
+
+class WikimediaIE(InfoExtractor):
+    _NETRC_MACHINE = 'wikimediaorg'
+    IE_NAME = 'wikimedia.org'
+    _API_BASE_URL = 'https://commons.wikimedia.org/'
+    _VALID_URL = r'https://commons.wikimedia.org/wiki/File:(?P<id>[^/]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        self.report_extraction(video_id)
+        video_url = self._html_search_regex(r'<source [^>]*src="([^"]+)"', webpage,
+                                            u'video URL')
+        resp = {}
+
+        subtitle_url = f'https://commons.wikimedia.org/w/api.php?' \
+                       f'action=timedtext&lang=nl&title=File%3A{urllib.parse.quote(video_id)}&trackformat=srt'
+        with open(video_id + '.srt', 'w+', encoding='utf') as f:
+            subtitles = requests.post(subtitle_url).text
+            if 'timedtext-notfound' not in subtitles:
+                f.write(subtitles)
+            else:
+                print("subtitles not found")
+        resp['url'] = video_url
+        resp['ext'] = 'webm'
+        resp['id'] = video_id
+        resp['title'] = video_id
+        return [resp]

From ecc770b99eabfe0c8677a59da4b14f4f4b643cfe Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Thu, 31 Mar 2022 15:19:00 +0500
Subject: [PATCH 159/394] adding https://commons.wikimedia.org to youtube-dl

---
 youtube_dl/extractor/wikimedia.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index fec867a122c..290db6baf0b 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -10,6 +10,14 @@ class WikimediaIE(InfoExtractor):
     _API_BASE_URL = 'https://commons.wikimedia.org/'
     _VALID_URL = r'https://commons.wikimedia.org/wiki/File:(?P<id>[^/]+)'
 
+    _TEST = {
+        'url': 'https://upload.wikimedia.org/wikipedia/commons/transcoded/d/d7/Die_Temperaturkurve_der_Erde_%28ZDF'
+               '%2C_Terra_X%29_720p_HD_50FPS.webm/Die_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS'
+               '.webm.480p.vp9.webm',
+        'ext': 'webm', 'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
+        'title': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
+        'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.'}
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -18,6 +26,8 @@ def _real_extract(self, url):
         self.report_extraction(video_id)
         video_url = self._html_search_regex(r'<source [^>]*src="([^"]+)"', webpage,
                                             u'video URL')
+        licenze = self._html_search_regex(f"(?<=td>This)(.*)(?=license.)", webpage, u'video license')
+        licenze = "This " + licenze + " license."
         resp = {}
 
         subtitle_url = f'https://commons.wikimedia.org/w/api.php?' \
@@ -32,4 +42,5 @@ def _real_extract(self, url):
         resp['ext'] = 'webm'
         resp['id'] = video_id
         resp['title'] = video_id
+        resp['license'] = licenze
         return [resp]

From e0dcf1db42e36113e638db7fbf669a2827b41acf Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Thu, 31 Mar 2022 15:21:38 +0500
Subject: [PATCH 160/394] adding https://commons.wikimedia.org to youtube-dl

---
 youtube_dl/extractor/wikimedia.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 290db6baf0b..0cf538464d3 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -32,12 +32,14 @@ def _real_extract(self, url):
 
         subtitle_url = f'https://commons.wikimedia.org/w/api.php?' \
                        f'action=timedtext&lang=nl&title=File%3A{urllib.parse.quote(video_id)}&trackformat=srt'
-        with open(video_id + '.srt', 'w+', encoding='utf') as f:
-            subtitles = requests.post(subtitle_url).text
-            if 'timedtext-notfound' not in subtitles:
+
+        subtitles = requests.post(subtitle_url).text
+        if 'timedtext-notfound' not in subtitles:
+            with open(video_id + '.srt', 'w+', encoding='utf') as f:
                 f.write(subtitles)
-            else:
-                print("subtitles not found")
+        else:
+            print("subtitles not found")
+
         resp['url'] = video_url
         resp['ext'] = 'webm'
         resp['id'] = video_id

From 7bf92fd726f4ebaf4752831263d91b8ef9fb0987 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Fri, 1 Apr 2022 03:09:52 +0500
Subject: [PATCH 161/394] adding https://commons.wikimedia.org to youtube-dl
 with url author description extension id title license

---
 youtube_dl/extractor/wikimedia.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 0cf538464d3..6dc558b719a 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -2,6 +2,7 @@
 import re
 import requests
 import urllib.parse
+from ..utils import clean_html
 
 
 class WikimediaIE(InfoExtractor):
@@ -14,20 +15,39 @@ class WikimediaIE(InfoExtractor):
         'url': 'https://upload.wikimedia.org/wikipedia/commons/transcoded/d/d7/Die_Temperaturkurve_der_Erde_%28ZDF'
                '%2C_Terra_X%29_720p_HD_50FPS.webm/Die_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS'
                '.webm.480p.vp9.webm',
+        'description': 'Deutsch:  Beschreibung auf der Seite: "Im Verlauf der Erdgeschichte glich das Klima einer '
+                       'Achterbahnfahrt. Die „Fieberkurve“ unseres Planeten zeigt die globalen Temperaturschwankungen '
+                       'bis heute – rekonstruiert anhand von historischen Klimadaten."\nZu Wikimedia Commons '
+                       'hochgeladen von: PantheraLeo1359531.\nHinweise zur Weiterverwendung: '
+                       'https://www.zdf.de/dokumentation/terra-x/terra-x-creative-commons-cc-100.html'
+                       '.\nVereinfachender Verlauf in der Geschichte der Erde, für die Zukunft spätestens ab dem Jahr '
+                       '2050 mit spekulativem Verlauf in der Prognose (ausgeprägtes Global-warming-Szenario ist '
+                       'dargestellt).English:  Climate change, Temperature in history of Earth, Video of Terra X.',
         'ext': 'webm', 'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
-        'title': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
-        'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.'}
+        'title': 'File:Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
+        'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.',
+        'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy'}
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        if not video_id.endswith('.webm'):
+            raise Exception("invalid video url")
+
         webpage = self._download_webpage(url, video_id)
         self.report_extraction(video_id)
         video_url = self._html_search_regex(r'<source [^>]*src="([^"]+)"', webpage,
                                             u'video URL')
         licenze = self._html_search_regex(f"(?<=td>This)(.*)(?=license.)", webpage, u'video license')
         licenze = "This " + licenze + " license."
+
+        description = self._html_search_regex(f'(?<=<div class="description mw-content-ltr de" dir="ltr" lang="de">)('
+                                              f'.*)(?=</div>)', webpage,
+                                              u'video description')
+
+        author = re.search(r'<td>([^\<]*?)<\/td>', str(webpage))
+        author = clean_html(author.group(0))
         resp = {}
 
         subtitle_url = f'https://commons.wikimedia.org/w/api.php?' \
@@ -41,8 +61,10 @@ def _real_extract(self, url):
             print("subtitles not found")
 
         resp['url'] = video_url
+        resp['description'] = description
         resp['ext'] = 'webm'
         resp['id'] = video_id
-        resp['title'] = video_id
+        resp['title'] = self._og_search_title(webpage)
         resp['license'] = licenze
+        resp['author'] = author
         return [resp]

From 0ae2a8f32056ecc3a8fb3c8fc9d36ac33066b7b3 Mon Sep 17 00:00:00 2001
From: alyssadev <alyssa.dev.smith@gmail.com>
Date: Fri, 1 Apr 2022 10:11:08 +1000
Subject: [PATCH 162/394] [nbl] addressed comments, added more test cases

---
 youtube_dl/extractor/nbl.py | 39 ++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py
index da6cf492d8e..1e0ffe4d69b 100644
--- a/youtube_dl/extractor/nbl.py
+++ b/youtube_dl/extractor/nbl.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
 
 class NBLIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P<id>[0-9]+)/(?P<display_id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)?/?(?P<id>[0-9]+)/?(?P<display_id>.*)'
     _TESTS = [
         {
             'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights',
@@ -16,6 +18,7 @@ class NBLIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Perth Wildcats vs. Sydney Kings - Game Highlights',
                 'description': 'Watch the Game Highlights from Perth Wildcats vs. Sydney Kings, 03/26/2022',
+                'display_id': 'Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights',
                 'upload_date': '20220326'
             }
         },
@@ -27,17 +30,29 @@ class NBLIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Perth Wildcats vs. Sydney Kings - Condensed Game',
                 'description': 'Watch the Condensed Game from Perth Wildcats vs. Sydney Kings, 03/26/2022',
+                'display_id': 'Perth-Wildcats-vs.-Sydney-Kings---Condensed-Game',
                 'upload_date': '20220326'
             }
         },
         {
             'url': 'https://nbl.com.au/tv/game-replays/1303323/NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings',
             'md5': '8505f02156f756e865a1aa80050eb768',
-            'expected_warnings': ['unable to extract OpenGraph description.+'],
             'info_dict': {
                 'id': '1303323',
                 'ext': 'mp4',
                 'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings',
+                'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings',
+                'upload_date': '20220326'
+            }
+        },
+        {
+            'url': 'https://nbl.com.au/tv/game-replays/1303323',
+            'md5': '8505f02156f756e865a1aa80050eb768',
+            'info_dict': {
+                'id': '1303323',
+                'ext': 'mp4',
+                'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings',
+                'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings',
                 'upload_date': '20220326'
             }
         }
@@ -45,25 +60,27 @@ class NBLIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
+        title = self._html_search_regex(r'<title\b[^>]*>\s*(.+?)\s*</title>', webpage, 'title')
         stream_url = self._download_json(
-            "https://ott.nbl.com.au/api/v2/content/" + video_id + "/access/hls",
+            'https://ott.nbl.com.au/api/v2/content/' + video_id + '/access/hls',
             video_id,
-            data=b"",
+            data=b'',
             headers={
-                "Referer": "https://ott.nbl.com.au/en-int/embed/" + video_id,
-                "Origin": "https://ott.nbl.com.au",
-                "Content-Length": "0"
+                'Referer': 'https://ott.nbl.com.au/en-int/embed/' + video_id,
+                'Origin': 'https://ott.nbl.com.au',
+                'Content-Length': '0'
             }
-        )["data"]["stream"]
-        formats = self._extract_m3u8_formats(stream_url, video_id, ext="mp4")
+        )['data']['stream']
+        formats = self._extract_m3u8_formats(stream_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
             'title': title,
             'description': self._og_search_description(webpage, default=None),
-            'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date')),
+            'display_id': mobj.groupdict().get('display_id') or title.replace(" ", "-"),
+            'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False)),
             'formats': formats
         }

From bf428499a017d5021f24057e14fb36838062505a Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Fri, 1 Apr 2022 05:21:57 +0500
Subject: [PATCH 163/394] made changes according to request also I really
 appreciate help by @dirkf

---
 youtube_dl/extractor/wikimedia.py | 53 ++++++++++++-------------------
 1 file changed, 20 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 6dc558b719a..0572760be65 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,8 +1,5 @@
 from .common import InfoExtractor
-import re
-import requests
-import urllib.parse
-from ..utils import clean_html
+from ..utils import get_element_by_class, compat_urlparse
 
 
 class WikimediaIE(InfoExtractor):
@@ -29,8 +26,7 @@ class WikimediaIE(InfoExtractor):
         'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy'}
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         if not video_id.endswith('.webm'):
             raise Exception("invalid video url")
@@ -39,32 +35,23 @@ def _real_extract(self, url):
         self.report_extraction(video_id)
         video_url = self._html_search_regex(r'<source [^>]*src="([^"]+)"', webpage,
                                             u'video URL')
-        licenze = self._html_search_regex(f"(?<=td>This)(.*)(?=license.)", webpage, u'video license')
+        licenze = self._html_search_regex(r'\bThis\s*(.*?)\s*license\b', webpage, u'video license')
         licenze = "This " + licenze + " license."
 
-        description = self._html_search_regex(f'(?<=<div class="description mw-content-ltr de" dir="ltr" lang="de">)('
-                                              f'.*)(?=</div>)', webpage,
-                                              u'video description')
-
-        author = re.search(r'<td>([^\<]*?)<\/td>', str(webpage))
-        author = clean_html(author.group(0))
-        resp = {}
-
-        subtitle_url = f'https://commons.wikimedia.org/w/api.php?' \
-                       f'action=timedtext&lang=nl&title=File%3A{urllib.parse.quote(video_id)}&trackformat=srt'
-
-        subtitles = requests.post(subtitle_url).text
-        if 'timedtext-notfound' not in subtitles:
-            with open(video_id + '.srt', 'w+', encoding='utf') as f:
-                f.write(subtitles)
-        else:
-            print("subtitles not found")
-
-        resp['url'] = video_url
-        resp['description'] = description
-        resp['ext'] = 'webm'
-        resp['id'] = video_id
-        resp['title'] = self._og_search_title(webpage)
-        resp['license'] = licenze
-        resp['author'] = author
-        return [resp]
+        description = get_element_by_class('description', webpage)
+
+        author = self._html_search_regex(r'<td>([^\<]*?)<\/td>', str(webpage), u"video author")
+
+        info = {}
+        subtitles = 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File%3A{}' \
+                    '&trackformat=srt'.format(compat_urlparse.quote_plus(video_id))
+        info['url'] = video_url
+        info['description'] = description
+        info['ext'] = 'webm'
+        info['id'] = video_id
+        info['title'] = self._og_search_title(webpage).replace("File:", "")
+        info['license'] = licenze
+        info['author'] = author
+        info['subtitles'] = {"nl": [{"ext": "srt", "url": subtitles}]}
+        print("ih")
+        return info

From 810c1a56a04fab4daa2bdf1bfe888da2e976143c Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Fri, 1 Apr 2022 05:25:00 +0500
Subject: [PATCH 164/394] made changes according to request also I really
 appreciate help by @dirkf

---
 youtube_dl/extractor/wikimedia.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 0572760be65..ef2536d0d3a 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -39,9 +39,7 @@ def _real_extract(self, url):
         licenze = "This " + licenze + " license."
 
         description = get_element_by_class('description', webpage)
-
         author = self._html_search_regex(r'<td>([^\<]*?)<\/td>', str(webpage), u"video author")
-
         info = {}
         subtitles = 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File%3A{}' \
                     '&trackformat=srt'.format(compat_urlparse.quote_plus(video_id))
@@ -53,5 +51,4 @@ def _real_extract(self, url):
         info['license'] = licenze
         info['author'] = author
         info['subtitles'] = {"nl": [{"ext": "srt", "url": subtitles}]}
-        print("ih")
         return info

From 42b0ca233da69291f63006b4f8be3abb94a73aca Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Fri, 1 Apr 2022 05:30:38 +0500
Subject: [PATCH 165/394] cleaned html tags inside description.

---
 youtube_dl/extractor/wikimedia.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index ef2536d0d3a..8478dbbc461 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import get_element_by_class, compat_urlparse
+from ..utils import get_element_by_class, compat_urlparse, clean_html
 
 
 class WikimediaIE(InfoExtractor):
@@ -21,9 +21,12 @@ class WikimediaIE(InfoExtractor):
                        '2050 mit spekulativem Verlauf in der Prognose (ausgeprägtes Global-warming-Szenario ist '
                        'dargestellt).English:  Climate change, Temperature in history of Earth, Video of Terra X.',
         'ext': 'webm', 'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
-        'title': 'File:Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
+        'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
         'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.',
-        'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy'}
+        'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy', 'subtitles': {'nl': [
+            {'ext': 'srt',
+             'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File'
+                    '%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&trackformat=srt'}]}}
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -44,7 +47,7 @@ def _real_extract(self, url):
         subtitles = 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File%3A{}' \
                     '&trackformat=srt'.format(compat_urlparse.quote_plus(video_id))
         info['url'] = video_url
-        info['description'] = description
+        info['description'] = clean_html(description)
         info['ext'] = 'webm'
         info['id'] = video_id
         info['title'] = self._og_search_title(webpage).replace("File:", "")

From 819a5f30a6d977d7a3e2476e2b5db3f798d262e0 Mon Sep 17 00:00:00 2001
From: alyssadev <alyssa.dev.smith@gmail.com>
Date: Fri, 1 Apr 2022 10:32:17 +1000
Subject: [PATCH 166/394] [nbl] added support for ott urls

---
 youtube_dl/extractor/nbl.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py
index 1e0ffe4d69b..464f40200ff 100644
--- a/youtube_dl/extractor/nbl.py
+++ b/youtube_dl/extractor/nbl.py
@@ -8,7 +8,7 @@
 
 
 class NBLIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)?/?(?P<id>[0-9]+)/?(?P<display_id>.*)'
+    _VALID_URL = r'https?://(?:www|ott\.)?nbl\.com\.au/(?:tv|en(?:-int)?)/(?P<section>game-replays|highlights|condensed-games|embed)?/?(?P<id>[0-9]+)/?(?P<display_id>.*)'
     _TESTS = [
         {
             'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights',
@@ -55,13 +55,26 @@ class NBLIE(InfoExtractor):
                 'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings',
                 'upload_date': '20220326'
             }
+        },
+        {
+            'url': 'https://ott.nbl.com.au/en-int/embed/1303323',
+            'md5': '8505f02156f756e865a1aa80050eb768',
+            'info_dict': {
+                'id': '1303323',
+                'ext': 'mp4',
+                'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings',
+                'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings'
+            }
         }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
-        webpage = self._download_webpage(url, video_id)
+        if mobj.groupdict().get("section") not in ['game-replays', 'highlights', 'condensed-games']:
+            webpage = self._download_webpage('https://ott.nbl.com.au/en-int/embed/' + video_id, video_id)
+        else:
+            webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title\b[^>]*>\s*(.+?)\s*</title>', webpage, 'title')
         stream_url = self._download_json(
@@ -81,6 +94,6 @@ def _real_extract(self, url):
             'title': title,
             'description': self._og_search_description(webpage, default=None),
             'display_id': mobj.groupdict().get('display_id') or title.replace(" ", "-"),
-            'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False)),
+            'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False, default=None)),
             'formats': formats
         }

From aeb5abe2b9d2512d39476d6ee292f9c0dc6f7fa1 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Sat, 2 Apr 2022 01:02:09 +0500
Subject: [PATCH 167/394] video name fixed.

---
 youtube_dl/extractor/wikimedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 8478dbbc461..339f7f8c053 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -49,7 +49,7 @@ def _real_extract(self, url):
         info['url'] = video_url
         info['description'] = clean_html(description)
         info['ext'] = 'webm'
-        info['id'] = video_id
+        info['id'] = video_id[:-5]
         info['title'] = self._og_search_title(webpage).replace("File:", "")
         info['license'] = licenze
         info['author'] = author

From 6dfb8d4ec6786248bb6e2d833aaa41f616be0724 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Sat, 2 Apr 2022 14:31:52 +0500
Subject: [PATCH 168/394] subtitles extraction for all language fixed.

---
 youtube_dl/extractor/wikimedia.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 339f7f8c053..b256c68793f 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,5 +1,6 @@
 from .common import InfoExtractor
 from ..utils import get_element_by_class, compat_urlparse, clean_html
+import re
 
 
 class WikimediaIE(InfoExtractor):
@@ -44,8 +45,7 @@ def _real_extract(self, url):
         description = get_element_by_class('description', webpage)
         author = self._html_search_regex(r'<td>([^\<]*?)<\/td>', str(webpage), u"video author")
         info = {}
-        subtitles = 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File%3A{}' \
-                    '&trackformat=srt'.format(compat_urlparse.quote_plus(video_id))
+
         info['url'] = video_url
         info['description'] = clean_html(description)
         info['ext'] = 'webm'
@@ -53,5 +53,12 @@ def _real_extract(self, url):
         info['title'] = self._og_search_title(webpage).replace("File:", "")
         info['license'] = licenze
         info['author'] = author
-        info['subtitles'] = {"nl": [{"ext": "srt", "url": subtitles}]}
+
+        subtitles = re.findall(r'\bsrc=\"\/w\/api\s*(.*?)\s*srt\b', str(webpage))
+        info['subtitles'] = {}
+        for sub in subtitles:
+            sub = 'https://commons.wikimedia.org/w/api' + sub + 'srt'
+            lang = sub[sub.find('lang=') + 5:]
+            lang = lang[:lang.find('&')]
+            info['subtitles'][lang] = [{"ext": "srt", "url": sub}]
         return info

From 31cd651d8ecb4894730c5d0717d7684b24c9dc6b Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 2 Apr 2022 15:37:50 +0200
Subject: [PATCH 169/394] update

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9314abe9d7..8f5cc103eb0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1164,6 +1164,7 @@
  - **Weibo**
  - **WeiboMobile**
  - **WeiqiTV**: WQTV
+ - **wikimedia.org**
  - **Wistia**
  - **WistiaPlaylist**
  - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ad32023dbf5..e7d3a198e39 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.13'
+__version__ = '2021.04.02'

From 8f48f187f8496d05590295705b4b533697508399 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Sun, 3 Apr 2022 01:31:15 +0500
Subject: [PATCH 170/394] subtitles extraction for all language fixed bad
 request

---
 youtube_dl/extractor/wikimedia.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index b256c68793f..c864563b610 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -60,5 +60,6 @@ def _real_extract(self, url):
             sub = 'https://commons.wikimedia.org/w/api' + sub + 'srt'
             lang = sub[sub.find('lang=') + 5:]
             lang = lang[:lang.find('&')]
+            sub = sub.replace(';','&')
             info['subtitles'][lang] = [{"ext": "srt", "url": sub}]
         return info

From fad516e22e58763ce0feab1034cc0c7aa2da184c Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Fri, 1 Apr 2022 18:12:21 +0900
Subject: [PATCH 171/394] [Tver] Fix extractor

    yt-dlp PR https://github.com/yt-dlp/yt-dlp/pull/3268

    Back-port to yt-dl
    Fix tests

    Co-authored-by: dirkf
---
 youtube_dl/extractor/tver.py | 229 ++++++++++++++++++++++++++++++-----
 1 file changed, 200 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/tver.py b/youtube_dl/extractor/tver.py
index a4a30b1e63e..3519c2321d4 100644
--- a/youtube_dl/extractor/tver.py
+++ b/youtube_dl/extractor/tver.py
@@ -4,58 +4,229 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_kwargs,
+    compat_str,
+)
 from ..utils import (
+    ExtractorError,
     int_or_none,
-    remove_start,
     smuggle_url,
-    try_get,
+    str_or_none,
 )
 
+try:
+    from ..utils import traverse_obj
+except ImportError:
+    from ..compat import compat_collections_abc
+
+    def traverse_obj(obj, *path_list, **kw):
+        ''' Traverse nested list/dict/tuple'''
+
+        # parameter defaults
+        default = kw.get('default')
+        expected_type = kw.get('expected_type')
+        get_all = kw.get('get_all', True)
+        casesense = kw.get('casesense', True)
+        is_user_input = kw.get('is_user_input', False)
+        traverse_string = kw.get('traverse_string', False)
+
+        def variadic(x, allowed_types=(compat_str, bytes)):
+            return x if isinstance(x, compat_collections_abc.Iterable) and not isinstance(x, allowed_types) else (x,)
+
+        def listish(l):
+            return isinstance(l, (list, tuple))
+
+        def from_iterable(iterables):
+            # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
+            for it in iterables:
+                for element in it:
+                    yield element
+
+        class Nonlocal:
+            pass
+        nl = Nonlocal()
+
+        if not casesense:
+            _lower = lambda k: (k.lower() if isinstance(k, compat_str) else k)
+            path_list = (map(_lower, variadic(path)) for path in path_list)
+
+        def _traverse_obj(obj, path, _current_depth=0):
+            path = tuple(variadic(path))
+            for i, key in enumerate(path):
+                if obj is None:
+                    return None
+                if listish(key):
+                    obj = [_traverse_obj(obj, sub_key, _current_depth) for sub_key in key]
+                    key = Ellipsis
+                if key is Ellipsis:
+                    obj = (obj.values() if isinstance(obj, dict)
+                           else obj if listish(obj)
+                           else compat_str(obj) if traverse_string else [])
+                    _current_depth += 1
+                    nl.depth = max(nl.depth, _current_depth)
+                    return [_traverse_obj(inner_obj, path[i + 1:], _current_depth) for inner_obj in obj]
+                elif callable(key):
+                    if listish(obj):
+                        obj = enumerate(obj)
+                    elif isinstance(obj, dict):
+                        obj = obj.items()
+                    else:
+                        if not traverse_string:
+                            return None
+                        obj = str(obj)
+                    _current_depth += 1
+                    nl.depth = max(nl.depth, _current_depth)
+                    return [_traverse_obj(v, path[i + 1:], _current_depth) for k, v in obj if key(k)]
+                elif isinstance(obj, dict) and not (is_user_input and key == ':'):
+                    obj = (obj.get(key) if casesense or (key in obj)
+                           else next((v for k, v in obj.items() if _lower(k) == key), None))
+                else:
+                    if is_user_input:
+                        key = (int_or_none(key) if ':' not in key
+                               else slice(*map(int_or_none, key.split(':'))))
+                        if key == slice(None):
+                            return _traverse_obj(obj, tuple([Ellipsis] + list(path[i + 1:])), _current_depth)
+                    if not isinstance(key, (int, slice)):
+                        return None
+                    if not listish(obj):
+                        if not traverse_string:
+                            return None
+                        obj = compat_str(obj)
+                    try:
+                        obj = obj[key]
+                    except IndexError:
+                        return None
+            return obj
+
+        if isinstance(expected_type, type):
+            type_test = lambda val: val if isinstance(val, expected_type) else None
+        elif expected_type is not None:
+            type_test = expected_type
+        else:
+            type_test = lambda val: val
+
+        for path in path_list:
+            nl.depth = 0
+            val = _traverse_obj(obj, path)
+            if val is not None:
+                if nl.depth:
+                    for _ in range(nl.depth - 1):
+                        val = from_iterable(v for v in val if v is not None)
+                    val = [v for v in map(type_test, val) if v is not None]
+                    if val:
+                        return val if get_all else val[0]
+                else:
+                    val = type_test(val)
+                    if val is not None:
+                        return val
+        return default
+
 
 class TVerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tver\.jp/(?P<path>(?:corner|episode|feature)/(?P<id>f?\d+))'
+    _VALID_URL = r'https?://(?:www\.)?tver\.jp/(?:(?P<type>lp|corner|series|episodes?|feature|tokyo2020/video)/)+(?P<id>[a-zA-Z0-9]+)'
     # videos are only available for 7 days
     _TESTS = [{
-        'url': 'https://tver.jp/corner/f0062178',
-        'only_matching': True,
+        'skip': 'videos are only available for 7 days',
+        'url': 'https://tver.jp/episodes/ephss8yveb',
+        'info_dict': {
+            'id': 'ref:af03de03-21ac-4b98-a53b-9ffd9b102e92',
+            'ext': 'mp4',
+            'title': '#44　料理と値段と店主にびっくり　オモてなしすぎウマい店　2時間SP',
+            'description': 'md5:66985373a66fed8ad3cd595a3cfebb13',
+            'upload_date': '20220329',
+            'uploader_id': '4394098882001',
+            'timestamp': 1648527032,
+        },
+        'add_ie': ['BrightcoveNew'],
     }, {
-        'url': 'https://tver.jp/feature/f0062413',
-        'only_matching': True,
+        'skip': 'videos are only available for 7 days',
+        'url': 'https://tver.jp/lp/episodes/ep6f16g26p',
+        'info_dict': {
+            'id': '6302378806001',
+            'ext': 'mp4',
+            # "April 11 (Mon) 23:06-Broadcast scheduled": sorry but this is "correct"
+            'title': '4月11日(月)23時06分 ~ 放送予定',
+            'description': 'md5:4029cc5f4b1e8090dfc5b7bd2bc5cd0b',
+            'upload_date': '20220331',
+            'uploader_id': '3971130137001',
+            'timestamp': 1648696456,
+        },
+        'add_ie': ['BrightcoveNew'],
     }, {
-        'url': 'https://tver.jp/episode/79622438',
+        'url': 'https://tver.jp/corner/f0103888',
         'only_matching': True,
     }, {
-        # subtitle = ' '
-        'url': 'https://tver.jp/corner/f0068870',
+        'url': 'https://tver.jp/lp/f0033031',
         'only_matching': True,
     }]
-    _TOKEN = None
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+    _PLATFORM_UID = None
+    _PLATFORM_TOKEN = None
+
+    def _download_json(self, url_or_request, video_id, **kwargs):
+
+        headers = {
+            'Origin': 'https://s.tver.jp',
+            'Referer': 'https://s.tver.jp/',
+        }
+        headers.update(kwargs.get('headers', {}))
+        kwargs.update(compat_kwargs({'headers': headers, }))
+
+        return super(TVerIE, self)._download_json(url_or_request, video_id, **kwargs)
 
     def _real_initialize(self):
-        self._TOKEN = self._download_json(
-            'https://tver.jp/api/access_token.php', None)['token']
+        create_response = self._download_json(
+            'https://platform-api.tver.jp/v2/api/platform_users/browser/create', None,
+            note='Creating session', data=b'device_type=pc', headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+        self._PLATFORM_UID = traverse_obj(create_response, ('result', 'platform_uid'), expected_type=compat_str)
+        self._PLATFORM_TOKEN = traverse_obj(create_response, ('result', 'platform_token'), expected_type=compat_str)
 
     def _real_extract(self, url):
-        path, video_id = re.match(self._VALID_URL, url).groups()
-        main = self._download_json(
-            'https://api.tver.jp/v4/' + path, video_id,
-            query={'token': self._TOKEN})['main']
-        p_id = main['publisher_id']
-        service = remove_start(main['service'], 'ts_')
-
-        r_id = main['reference_id']
-        if service not in ('tx', 'russia2018', 'sebare2018live', 'gorin'):
+        video_id = self._match_id(url)
+
+        video_id, video_type = re.match(self._VALID_URL, url).group('id', 'type')
+        if video_type not in ('episode', 'episodes', 'series'):
+            webpage = self._download_webpage(url, video_id, note='Resolving to new URL')
+            video_id = self._match_id(self._search_regex(
+                (r'''canonical['"]\s+href\s*=\s*(?P<q>'|")(?P<url>https?://tver\.jp/(?!(?P=q)).+?)(?P=q)''',
+                 r'&link=(?P<url>https?://tver\.jp/(?!(?P=q)).+?)[?&]'),
+                webpage, 'url regex', group='url'))
+        video_info = self._download_json(
+            'https://statics.tver.jp/content/episode/{0}.json'.format(video_id), video_id,
+            query={'v': '5'})
+        p_id = traverse_obj(video_info, ('video', 'accountID'), expected_type=compat_str)
+        r_id = traverse_obj(video_info, ('video', ('videoRefID', 'videoID')), get_all=False, expected_type=compat_str)
+        if None in (p_id, r_id):
+            raise ExtractorError(
+                'Failed to extract '
+                + ', '.join(
+                    (x[0] for x in (('accountID', p_id), ('videoRefID', r_id), )
+                     if x[1] is None)),
+                expected=False)
+        if not r_id.isdigit():
             r_id = 'ref:' + r_id
-        bc_url = smuggle_url(
-            self.BRIGHTCOVE_URL_TEMPLATE % (p_id, r_id),
-            {'geo_countries': ['JP']})
+
+        additional_info = self._download_json(
+            'https://platform-api.tver.jp/service/api/v1/callEpisode/{0}?require_data=mylist,later[epefy106ur],good[epefy106ur],resume[epefy106ur]'.format(video_id),
+            video_id, fatal=False,
+            query={
+                'platform_uid': self._PLATFORM_UID,
+                'platform_token': self._PLATFORM_TOKEN,
+            }, headers={
+                'x-tver-platform-type': 'web'
+            })
 
         return {
             '_type': 'url_transparent',
-            'description': try_get(main, lambda x: x['note'][0]['text'], compat_str),
-            'episode_number': int_or_none(try_get(main, lambda x: x['ext']['episode_number'])),
-            'url': bc_url,
+            'title': str_or_none(video_info.get('title')),
+            'description': str_or_none(video_info.get('description')),
+            'url': smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % (p_id, r_id), {'geo_countries': ['JP']}),
+            'series': traverse_obj(
+                additional_info, ('result', ('episode', 'series'), 'content', ('seriesTitle', 'title')),
+                get_all=False, expected_type=compat_str),
             'ie_key': 'BrightcoveNew',
         }

From a10f613d62eacbd0ddedfcc2f70ee34d088be4a6 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Mon, 4 Apr 2022 17:47:05 +0500
Subject: [PATCH 172/394] fixed Test part for the wikimedia extractor

python test/test_download.py TestDownload.test_Wikimedia

[wikimedia.org] Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm: Downloading webpage
[wikimedia.org] Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm: Extracting information
[info] Writing video description metadata as JSON to: test_Wikimedia_Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.info.json
[debug] Invoking downloader on 'https://upload.wikimedia.org/wikipedia/commons/transcoded/d/d7/Die_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm/Die_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm.480p.vp9.webm'
[download] Destination: test_Wikimedia_Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm
[download] 100% of 10.00KiB in 00:00
.
----------------------------------------------------------------------
Ran 1 test in 2.262s

OK
---
 youtube_dl/extractor/wikimedia.py | 40 ++++++++++++++++---------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index c864563b610..b24410de2c9 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -10,24 +10,26 @@ class WikimediaIE(InfoExtractor):
     _VALID_URL = r'https://commons.wikimedia.org/wiki/File:(?P<id>[^/]+)'
 
     _TEST = {
-        'url': 'https://upload.wikimedia.org/wikipedia/commons/transcoded/d/d7/Die_Temperaturkurve_der_Erde_%28ZDF'
-               '%2C_Terra_X%29_720p_HD_50FPS.webm/Die_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS'
-               '.webm.480p.vp9.webm',
-        'description': 'Deutsch:  Beschreibung auf der Seite: "Im Verlauf der Erdgeschichte glich das Klima einer '
-                       'Achterbahnfahrt. Die „Fieberkurve“ unseres Planeten zeigt die globalen Temperaturschwankungen '
-                       'bis heute – rekonstruiert anhand von historischen Klimadaten."\nZu Wikimedia Commons '
-                       'hochgeladen von: PantheraLeo1359531.\nHinweise zur Weiterverwendung: '
-                       'https://www.zdf.de/dokumentation/terra-x/terra-x-creative-commons-cc-100.html'
-                       '.\nVereinfachender Verlauf in der Geschichte der Erde, für die Zukunft spätestens ab dem Jahr '
-                       '2050 mit spekulativem Verlauf in der Prognose (ausgeprägtes Global-warming-Szenario ist '
-                       'dargestellt).English:  Climate change, Temperature in history of Earth, Video of Terra X.',
-        'ext': 'webm', 'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
-        'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
-        'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.',
-        'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy', 'subtitles': {'nl': [
-            {'ext': 'srt',
-             'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&lang=nl&title=File'
-                    '%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&trackformat=srt'}]}}
+        'url': 'https://commons.wikimedia.org/wiki/File:Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
+        'info_dict': {
+            'description': 'Deutsch:  Beschreibung auf der Seite: "Im Verlauf der Erdgeschichte glich das Klima einer Achterbahnfahrt. Die „Fieberkurve“ unseres Planeten zeigt die globalen Temperaturschwankungen bis heute – rekonstruiert anhand von historischen Klimadaten."\nZu Wikimedia Commons hochgeladen von: PantheraLeo1359531.\nHinweise zur Weiterverwendung: https://www.zdf.de/dokumentation/terra-x/terra-x-creative-commons-cc-100.html.\nVereinfachender Verlauf in der Geschichte der Erde, für die Zukunft spätestens ab dem Jahr 2050 mit spekulativem Verlauf in der Prognose (ausgeprägtes Global-warming-Szenario ist dargestellt).English:  Climate change, Temperature in history of Earth, Video of Terra X.',
+            'ext': 'webm',
+            'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
+            'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
+            'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.',
+            'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy',
+            'subtitles': {'de': [
+                {'ext': 'srt',
+                 'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=de&amp&trackformat=srt'}],
+                'en-gb': [
+                    {'ext': 'srt',
+                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=en-gb&amp&trackformat=srt'}],
+                'nl': [
+                    {'ext': 'srt',
+                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=nl&amp&trackformat=srt'}
+                ]}
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -60,6 +62,6 @@ def _real_extract(self, url):
             sub = 'https://commons.wikimedia.org/w/api' + sub + 'srt'
             lang = sub[sub.find('lang=') + 5:]
             lang = lang[:lang.find('&')]
-            sub = sub.replace(';','&')
+            sub = sub.replace(';', '&')
             info['subtitles'][lang] = [{"ext": "srt", "url": sub}]
         return info

From 961a3f3d8b2d88954ac3b63c0c36c4bf2d78533f Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Tue, 5 Apr 2022 05:30:07 +0500
Subject: [PATCH 173/394] fixed added support for multiple video format.

---
 youtube_dl/extractor/wikimedia.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index b24410de2c9..e5b6c32738d 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import get_element_by_class, compat_urlparse, clean_html
+from ..utils import get_element_by_class, determine_ext, clean_html, KNOWN_EXTENSIONS
 import re
 
 
@@ -33,8 +33,8 @@ class WikimediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        if not video_id.endswith('.webm'):
+        ext = determine_ext(url)
+        if not ext.lower() in KNOWN_EXTENSIONS:
             raise Exception("invalid video url")
 
         webpage = self._download_webpage(url, video_id)
@@ -50,8 +50,8 @@ def _real_extract(self, url):
 
         info['url'] = video_url
         info['description'] = clean_html(description)
-        info['ext'] = 'webm'
-        info['id'] = video_id[:-5]
+        info['ext'] = ext
+        info['id'] = video_id.replace('.' + ext, "")
         info['title'] = self._og_search_title(webpage).replace("File:", "")
         info['license'] = licenze
         info['author'] = author

From 3f2d5391fdf673b0f4637a5b1fe0f304567d9bfb Mon Sep 17 00:00:00 2001
From: Time Lord <8444252+sxx1314@users.noreply.github.com>
Date: Wed, 6 Apr 2022 05:34:02 +0800
Subject: [PATCH 174/394] fix url

---
 youtube_dl/extractor/afreecatv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index b56abb1e604..ab1e382f893 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -26,7 +26,7 @@ class AfreecaTVIE(InfoExtractor):
                                 /app/(?:index|read_ucc_bbs)\.cgi|
                                 /player/[Pp]layer\.(?:swf|html)
                             )\?.*?\bnTitleNo=|
-                            vod\.afreecatv\.com/PLAYER/STATION/
+                            vod\.afreecatv\.com/player/
                         )
                         (?P<id>\d+)
                     '''

From 65712d99c46454a475f02337911a2cccaf7ece8a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 7 Apr 2022 23:08:42 +0100
Subject: [PATCH 175/394] Support Series page

---
 youtube_dl/extractor/rtve.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index f060ffdc9e1..638eddd4d20 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -9,7 +9,9 @@
 from .common import InfoExtractor
 from ..compat import (
     compat_b64decode,
+    compat_parse_qs,
     compat_struct_unpack,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     determine_ext,
@@ -25,8 +27,8 @@
 
 
 class RTVEALaCartaIE(InfoExtractor):
-    IE_NAME = 'rtve.es:alacarta'
-    IE_DESC = 'RTVE a la carta'
+    IE_NAME = 'rtve.es:play'
+    IE_DESC = 'RTVE Play'
     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?((alacarta|playz?)/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
@@ -90,6 +92,12 @@ class RTVEALaCartaIE(InfoExtractor):
     }, {
         'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.rtve.es/play/videos/modulos/capitulos/11332/?currentpage=pf_serie',
+        'info_dict': {
+            'id': '11332',
+        },
+        'playlist_mincount': 20,
     }]
 
     def _real_initialize(self):
@@ -164,8 +172,21 @@ def _extract_png_formats(self, video_id):
         self._sort_formats(formats)
         return formats
 
+    def _extract_playlist(self, url, playlist_id):
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        matches = re.findall(r'''<a\b[^>]*\bhref\s*=\s*["'](%s)''' % (self._VALID_URL, ), webpage)
+
+        return self.playlist_from_matches(matches, playlist_id=playlist_id, getter=lambda x: x[0], ie=self.ie_key())
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        if 'pf_serie' == qs.get('currentpage', [None])[-1]:
+            return self._extract_playlist(url, video_id)
+
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]

From 85653519e951c579502665a0a2f8183b089aeb77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Dobo=C5=A1?= <jakub.dobos@gab.sk>
Date: Sun, 10 Apr 2022 01:34:09 +0200
Subject: [PATCH 176/394] update docs away from freenode

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2841ed68ff3..0b2bec38369 100644
--- a/README.md
+++ b/README.md
@@ -1406,7 +1406,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
+Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.libera.chat/#youtube-dl) on freenode ([webchat](https://web.libera.chat/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```

From e942147a4fc442dc9b5e64f4de1b0262496141be Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sun, 10 Apr 2022 16:26:34 +0400
Subject: [PATCH 177/394] Updated Streamff and Added StreamWO

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamff.py   |  5 ++---
 youtube_dl/extractor/streamwo.py   | 31 ++++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 youtube_dl/extractor/streamwo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bfc9a4bbc2e..4e2f9687d5d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1195,6 +1195,7 @@
 from .streamcz import StreamCZIE
 from .streamff import StreamFFIE
 from .streetvoice import StreetVoiceIE
+from .streamwo import StreamwoIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
 from .sunporno import SunPornoIE
diff --git a/youtube_dl/extractor/streamff.py b/youtube_dl/extractor/streamff.py
index 0ea035ad20e..d0cd6722936 100644
--- a/youtube_dl/extractor/streamff.py
+++ b/youtube_dl/extractor/streamff.py
@@ -22,6 +22,5 @@ class StreamFFIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        response = self._download_json("https://streamff.com/api/videos/%s" % video_id, video_id)
-        external_link = response["externalLink"]
-        return self.url_result(external_link, ie="Generic")
+        video_url = "https://streamff.com/uploads/%s.mp4" % video_id
+        return self.url_result(video_url, ie="Generic")
diff --git a/youtube_dl/extractor/streamwo.py b/youtube_dl/extractor/streamwo.py
new file mode 100644
index 00000000000..fdc89eafc0d
--- /dev/null
+++ b/youtube_dl/extractor/streamwo.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamwoIE(InfoExtractor):
+    _VALID_URL = r'https?://streamwo\.com/(?P<id>\w+)'
+    _TEST = {
+        "url": 'https://streamwo.com/4529aff4',
+        'md5': 'eaf1f163635c868ecbba95d23ba83448',
+        'info_dict': {
+            'id': '4529aff4',
+            'ext': 'mp4',
+            'title': 'Goal 2021 13-04-50'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        self.report_extraction(video_id)
+        title = video_id
+        video_url = self._html_search_regex(r'<source src="(.+?)"', webpage, 'video URL')
+        video_url = video_url.replace("/../", "/")
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            "title": title
+        }

From fdd8f7edf736c5008cf276e99999a568e78fd171 Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Sun, 10 Apr 2022 18:00:26 +0400
Subject: [PATCH 178/394] Adde StreamGG

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamgg.py   | 31 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/streamgg.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4e2f9687d5d..1a8f823fe3b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1195,6 +1195,7 @@
 from .streamcz import StreamCZIE
 from .streamff import StreamFFIE
 from .streetvoice import StreetVoiceIE
+from .streamgg import StreamggIE
 from .streamwo import StreamwoIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamgg.py b/youtube_dl/extractor/streamgg.py
new file mode 100644
index 00000000000..a7ab85ca88f
--- /dev/null
+++ b/youtube_dl/extractor/streamgg.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamggIE(InfoExtractor):
+    _VALID_URL = r'https?://up.streamgg\.com/v/(?P<id>\w+)'
+    _TEST = {
+        "url": 'https://up.streamgg.com/v/r9gb3m95',
+        'md5': 'eaf1f163635c868ecbba95d23ba83448',
+        'info_dict': {
+            'id': '4529aff4',
+            'ext': 'mp4',
+            'title': 'Goal 2021 13-04-50'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        self.report_extraction(video_id)
+        title = video_id
+        video_url = self._html_search_regex(r'<source src="(.+?)"', webpage, 'video URL')
+        video_url = video_url.replace("/../", "/")
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            "title": title
+        }

From c278f31d62ec63ad110a934bcbffd9131af207a8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 10 Apr 2022 14:19:35 +0100
Subject: [PATCH 179/394] [Porn91] Update extractor

---
 youtube_dl/extractor/porn91.py | 57 +++++++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 20eac647a87..b299c7b9ad3 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,11 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
+    clean_html,
+    extract_attributes,
+    get_elements_by_class,
     parse_duration,
-    int_or_none,
     ExtractorError,
+    str_to_int,
+    strip_or_none,
+    unified_strdate,
 )
 
 
@@ -15,13 +25,15 @@ class Porn91IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-        'md5': '7fcdb5349354f40d41689bd0fa8db05a',
+        'md5': 'd869db281402e0ef4ddef3c38b866f86',
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
             'ext': 'mp4',
             'duration': 431,
             'age_limit': 18,
+            'upload_date': '20150520',
+            'uploader': '千岁九王爷',
         }
     }
 
@@ -35,29 +47,44 @@ def _real_extract(self, url):
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
 
-        title = self._search_regex(
-            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
-        title = title.replace('\n', '')
+        title = self._html_search_regex(
+            r'(?s)<title\b[^>]*>\s*(.+?)\s*(?:Chinese\s+homemade\s+video\s*)?</title', webpage, 'title')
 
         video_link_url = self._search_regex(
-            r'<textarea[^>]+id=["\']fm-video_link[^>]+>([^<]+)</textarea>',
-            webpage, 'video link')
-        videopage = self._download_webpage(video_link_url, video_id)
+            r'''document\s*\.\s*write\s*\(\s*strencode2\s*\((?P<q>"|')(?P<enc_str>[%\da-fA-F]+)(?P=q)\s*\)''',
+            webpage, 'video link', group='enc_str')
+        video_link_url = compat_urllib_parse_unquote(video_link_url)
 
-        info_dict = self._parse_html5_media_entries(url, videopage, video_id)[0]
+        info_dict = self._parse_html5_media_entries(url, '<video>%s</video>' % (video_link_url, ), video_id)[0]
 
-        duration = parse_duration(self._search_regex(
-            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+        FIELD_MAP = {
+            '时长': ('duration', parse_duration, 'Runtime', ),
+            '查看': ('view_count', str_to_int, 'Views', ),
+            '留言': ('comment_count', str_to_int, 'Comments', ),
+            '收藏': ('like_count', str_to_int, 'Favorites', ),
+            '添加时间': ('upload_date', unified_strdate, 'Added', ),
+            # same as title for en, not description for cn_CN
+            '__ignore__': ('description', strip_or_none, 'Description', ),
+            '作者': ('uploader', strip_or_none, 'From', ),
+        }
+        for elt in get_elements_by_class('info', re.sub(r'</span>\s*<span\b[^>]*?>', '', webpage)) or []:
+            elt = re.split(r':\s*', clean_html(elt), 1)
+            if len(elt) != 2 or elt[1] == '':
+                continue
+            parm = FIELD_MAP.get(elt[0].strip())
+            if parm and elt[1] is not None:
+                info_dict[parm[0]] = parm[1](elt[1]) if parm[1] else elt[1]
 
-        comment_count = int_or_none(self._search_regex(
-            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
+        thumbnail = extract_attributes(
+            self._search_regex(
+                r'''(<video\b[^>]*\bid\s*=\s*(?P<q>"|')player_one(?P=q)[^>]*>)''',
+                webpage, 'poster', default='')).get('poster')
 
         info_dict.update({
             'id': video_id,
             'title': title,
-            'duration': duration,
-            'comment_count': comment_count,
             'age_limit': self._rta_search(webpage),
+            'thumbnail': thumbnail,
         })
 
         return info_dict

From f70619bfe57d462a10bc4d4e5f94c20c89906314 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 12 Apr 2022 03:31:54 +0100
Subject: [PATCH 180/394] [YouTube] Initial implementation of clips

---
 youtube_dl/downloader/__init__.py |  3 ++
 youtube_dl/downloader/external.py |  4 ++
 youtube_dl/extractor/youtube.py   | 87 +++++++++++++++++++++++++------
 3 files changed, 78 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index d701d629221..f942ff3196c 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -64,6 +64,9 @@ def _get_suitable_downloader(info_dict, params={}):
     if protocol == 'm3u8_native' and params.get('hls_prefer_native') is False:
         return FFmpegFD
 
+    if params.get('_clip_args'):
+        return FFmpegFD
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index c31f8910ad8..c86a3b033d2 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -311,6 +311,10 @@ def _call_downloader(self, tmpfilename, info_dict):
             elif isinstance(conn, compat_str):
                 args += ['-rtmp_conn', conn]
 
+        ss = self.params.get('_clip_args', '').split(',')
+        if len(ss) == 2:
+            args += ['-ss', ss[0], '-to', ss[1]]
+
         args += ['-i', url, '-c', 'copy']
 
         if self.params.get('test', False):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 41695a561e5..887619341e8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1984,8 +1984,9 @@ def feed_entry(name):
             player_response,
             lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
         if pctr:
-            def process_language(container, base_url, lang_code, query):
-                lang_subs = []
+
+            def process_language(container, base_url, lang_code, sub_name, query):
+                lang_subs = container.setdefault(lang_code, [])
                 for fmt in self._SUBTITLE_FORMATS:
                     query.update({
                         'fmt': fmt,
@@ -1993,30 +1994,50 @@ def process_language(container, base_url, lang_code, query):
                     lang_subs.append({
                         'ext': fmt,
                         'url': update_url_query(base_url, query),
+                        'name': sub_name,
                     })
-                container[lang_code] = lang_subs
 
-            subtitles = {}
-            for caption_track in (pctr.get('captionTracks') or []):
+            def get_lang_code(track):
+                return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
+                        or track.get('languageCode'))
+
+            def get_text(x):
+                return try_get(x, (lambda x: x['simpleText'],
+                                   lambda x: x['runs'][0]['text']), compat_str)
+
+            subtitles, automatic_captions = {}, {}
+            for lang_code, caption_track in dict(
+                    (get_lang_code(sub), sub) for sub in (pctr.get('captionTracks') or [])).items():
                 base_url = caption_track.get('baseUrl')
                 if not base_url:
                     continue
+                lang_name = get_text(caption_track) or lang_code
                 if caption_track.get('kind') != 'asr':
-                    lang_code = caption_track.get('languageCode')
                     if not lang_code:
                         continue
                     process_language(
-                        subtitles, base_url, lang_code, {})
-                    continue
-                automatic_captions = {}
-                for translation_language in (pctr.get('translationLanguages') or []):
-                    translation_language_code = translation_language.get('languageCode')
-                    if not translation_language_code:
+                        subtitles, base_url, lang_code, lang_name, {})
+                    if not caption_track.get('isTranslatable'):
                         continue
-                    process_language(
-                        automatic_captions, base_url, translation_language_code,
-                        {'tlang': translation_language_code})
-                info['automatic_captions'] = automatic_captions
+                for trans_code, trans_name in dict(
+                        (lang.get('languageCode'), get_text(lang.get('languageName')))
+                        for lang in pctr.get('translationLanguages') or []).items():
+                    if not trans_code:
+                        continue
+                    if caption_track.get('kind') != 'asr':
+                        trans_code += '-' + lang_code
+                        trans_name += ' from ' + lang_name
+                    # Add "-orig" label to the original language so that it can be distinguished
+                    # The subs are returned without "-orig" as well for compatibility
+                    if lang_code == 'a-' + trans_code:
+                        process_language(
+                            automatic_captions, base_url, trans_code + '-orig', trans_name + ' (Original)', {})
+                    # Setting tlang=lang returns damaged subtitles.
+                    # Not using lang_code == f'a-{trans_code}' here for future-proofing
+                    orig_lang = parse_qs(base_url).get('lang', [None])[-1]
+                    process_language(automatic_captions, base_url, trans_code, trans_name,
+                                     {} if orig_lang == trans_code else {'tlang': trans_code})
+            info['automatic_captions'] = automatic_captions
             info['subtitles'] = subtitles
 
         parsed_url = compat_urllib_parse_urlparse(url)
@@ -3007,6 +3028,38 @@ def _extract_identity_token(self, ytcfg, webpage):
             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
             'identity token', default=None)
 
+    def _process_clip(self, url, data):
+        renderer = try_get(
+            data,
+            lambda x: next(
+                (r for r in (s.get('engagementPanelSectionListRenderer')
+                             for s in x['engagementPanels'])
+                 if (r or {}).get('panelIdentifier') == 'engagement-panel-clip-view'),
+                None), dict)
+        if not renderer:
+            return
+        clip_commands = try_get(
+            renderer,
+            lambda x: x['content']['clipSectionRenderer']['contents'][0]['clipAttributionRenderer']['onScrubExit']['commandExecutorCommand']['commands'],
+            list) or []
+        clip_open = try_get(
+            clip_commands,
+            lambda x: next((c['openPopupAction'] for c in x if c.get('openPopupAction')), None),
+            dict)
+        clip_open_commands = try_get(
+            clip_open,
+            lambda x: x['popup']['notificationActionRenderer']['actionButton']['buttonRenderer']['command']['commandExecutorCommand']['commands'],
+            list) or []
+        clip_data = try_get(
+            clip_open_commands,
+            lambda x: next((c['loopCommand'] for c in x if c.get('loopCommand')), None),
+            dict)
+        if clip_data.get('postId') in url:
+            clip_times = (int_or_none(clip_data.get('startTimeMs')),
+                          int_or_none(clip_data.get('endTimeMs')))
+            if None not in clip_times:
+                self._downloader.params['_clip_args'] = '%gms,%gms' % clip_times
+
     def _real_extract(self, url):
         item_id = self._match_id(url)
         url = compat_urlparse.urlunparse(
@@ -3036,6 +3089,8 @@ def _real_extract(self, url):
             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
             compat_str) or video_id
         if video_id:
+            if item_id == 'clip':
+                self._process_clip(url, data)
             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
         # Capture and output alerts
         alert = self._extract_alert(data)

From 10e00397633d0e839bf14621fc0195064aa80f9d Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Tue, 12 Apr 2022 18:31:50 +0200
Subject: [PATCH 181/394] verion 0412

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e7d3a198e39..e19d1dfa1e8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.02'
+__version__ = '2021.04.12'

From 2e19aac492daec451e6ca119110dca7e23bbc5f1 Mon Sep 17 00:00:00 2001
From: Dorian Westacott <dodrian@dodrian.com>
Date: Wed, 16 Mar 2022 18:54:20 -0500
Subject: [PATCH 182/394] Support multiple pages

From https://github.com/yt-dlp/yt-dlp/pull/3026

Authored by: dodrian
---
 youtube_dl/extractor/paramountplus.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/paramountplus.py b/youtube_dl/extractor/paramountplus.py
index 86c07f61f7b..67a04e5ff62 100644
--- a/youtube_dl/extractor/paramountplus.py
+++ b/youtube_dl/extractor/paramountplus.py
@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import itertools
 
 from .common import InfoExtractor
 from .cbs import CBSBaseIE
@@ -130,11 +131,13 @@ class ParamountPlusSeriesIE(InfoExtractor):
             'id': 'spongebob-squarepants',
         }
     }]
-    _API_URL = 'https://www.paramountplus.com/shows/{}/xhr/episodes/page/0/size/100000/xs/0/season/0/'
 
     def _entries(self, show_name):
-        show_json = self._download_json(self._API_URL.format(show_name), video_id=show_name)
-        if show_json.get('success'):
+        for page in itertools.count():
+            show_json = self._download_json(
+                'https://www.paramountplus.com/shows/{0}/xhr/episodes/page/{1}/size/50/xs/0/season/0'.format(show_name, page), show_name)
+            if not show_json.get('success'):
+                return
             for episode in show_json['result']['data']:
                 yield self.url_result(
                     'https://www.paramountplus.com%s' % episode['url'],

From 11fc1e228aeed255292c2994b0cc779de06d597c Mon Sep 17 00:00:00 2001
From: Sipherdrakon <64430430+Sipherdrakon@users.noreply.github.com>
Date: Fri, 18 Mar 2022 05:49:31 -0400
Subject: [PATCH 183/394] Change VALID_URL

From https://github.com/yt-dlp/yt-dlp/pull/3098

Authored by: Sipherdrakon
---
 youtube_dl/extractor/cbs.py           | 33 +++++++++++++++------------
 youtube_dl/extractor/paramountplus.py | 25 ++++++++++++++------
 2 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 8ff2b2240be..c4e901b429b 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .theplatform import ThePlatformFeedIE
@@ -77,21 +78,21 @@ class CBSIE(CBSBaseIE):
         (?:
             cbs:|
             https?://(?:www\.)?(?:
-                cbs\.com/(?:shows/[^/]+/video|movies/[^/]+)/|
+                cbs\.com/(?:shows|movies)/(?:video|[^/]+/video|[^/]+)/|
                 colbertlateshow\.com/(?:video|podcasts)/)
         )(?P<id>[\w-]+)'''
 
     # All tests are blocked outside US
     _TESTS = [{
-        'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
+        'url': 'https://www.cbs.com/shows/video/xrUyNLtl9wd8D_RWWAg9NU2F_V6QpB3R/',
         'info_dict': {
-            'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_',
+            'id': 'xrUyNLtl9wd8D_RWWAg9NU2F_V6QpB3R',
             'ext': 'mp4',
-            'title': 'Connect Chat feat. Garth Brooks',
-            'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
-            'duration': 1495,
-            'timestamp': 1385585425,
-            'upload_date': '20131127',
+            'title': 'Tough As Nails - Dreams Never Die',
+            'description': 'md5:a3535a62531cdd52b0364248a2c1ae33',
+            'duration': 2588,
+            'timestamp': 1639015200,
+            'upload_date': '20211209',
             'uploader': 'CBSI-NEW',
         },
         'params': {
@@ -100,14 +101,14 @@ class CBSIE(CBSBaseIE):
         },
         '_skip': 'Blocked outside the US',
     }, {
-        'url': 'https://www.cbs.com/shows/the-late-show-with-stephen-colbert/video/60icOhMb9NcjbcWnF_gub9XXHdeBcNk2/the-late-show-6-23-21-christine-baranski-joy-oladokun-',
+        'url': 'https://www.cbs.com/shows/video/sZH1MGgomIosZgxGJ1l263MFq16oMtW1/',
         'info_dict': {
-            'id': '60icOhMb9NcjbcWnF_gub9XXHdeBcNk2',
-            'title': 'The Late Show - 6/23/21 (Christine Baranski, Joy Oladokun)',
-            'timestamp': 1624507140,
-            'description': 'md5:e01af24e95c74d55e8775aef86117b95',
+            'id': 'sZH1MGgomIosZgxGJ1l263MFq16oMtW1',
+            'title': 'The Late Show - 3/16/22 (Michael Buble, Rose Matafeo)',
+            'timestamp': 1647488100,
+            'description': 'md5:d0e6ec23c544b7fa8e39a8e6844d2439',
             'uploader': 'CBSI-NEW',
-            'upload_date': '20210624',
+            'upload_date': '20220317',
         },
         'params': {
             'ignore_no_formats_error': True,
@@ -121,6 +122,10 @@ class CBSIE(CBSBaseIE):
     }, {
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
+    }, {
+        # old format
+        'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
+        'only_matching': True,
     }]
 
     def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
diff --git a/youtube_dl/extractor/paramountplus.py b/youtube_dl/extractor/paramountplus.py
index 67a04e5ff62..8b190890696 100644
--- a/youtube_dl/extractor/paramountplus.py
+++ b/youtube_dl/extractor/paramountplus.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 import itertools
 
@@ -14,12 +15,12 @@ class ParamountPlusIE(CBSBaseIE):
         (?:
             paramountplus:|
             https?://(?:www\.)?(?:
-                paramountplus\.com/(?:shows/[^/]+/video|movies/[^/]+)/
+                paramountplus\.com/(?:shows|movies)/(?:video|[^/]+/video|[^/]+)/
         )(?P<id>[\w-]+))'''
 
     # All tests are blocked outside US
     _TESTS = [{
-        'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/',
+        'url': 'https://www.paramountplus.com/shows/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/',
         'info_dict': {
             'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k',
             'ext': 'mp4',
@@ -34,7 +35,7 @@ class ParamountPlusIE(CBSBaseIE):
             'skip_download': 'm3u8',
         },
     }, {
-        'url': 'https://www.paramountplus.com/shows/tooning-out-the-news/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/7-23-21-week-in-review-rep-jahana-hayes-howard-fineman-sen-michael-bennet-sheera-frenkel-cecilia-kang-/',
+        'url': 'https://www.paramountplus.com/shows/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/',
         'info_dict': {
             'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd',
             'ext': 'mp4',
@@ -49,7 +50,7 @@ class ParamountPlusIE(CBSBaseIE):
             'skip_download': 'm3u8',
         },
     }, {
-        'url': 'https://www.paramountplus.com/movies/daddys-home/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC',
+        'url': 'https://www.paramountplus.com/movies/video/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC/',
         'info_dict': {
             'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC',
             'ext': 'mp4',
@@ -65,7 +66,7 @@ class ParamountPlusIE(CBSBaseIE):
         },
         'expected_warnings': ['Ignoring subtitle tracks'],  # TODO: Investigate this
     }, {
-        'url': 'https://www.paramountplus.com/movies/sonic-the-hedgehog/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc',
+        'url': 'https://www.paramountplus.com/movies/video/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc/',
         'info_dict': {
             'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc',
             'ext': 'mp4',
@@ -81,10 +82,20 @@ class ParamountPlusIE(CBSBaseIE):
         },
         'expected_warnings': ['Ignoring subtitle tracks'],
     }, {
-        'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/',
+        'url': 'https://www.paramountplus.com/shows/the-real-world/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/the-real-world-reunion/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.paramountplus.com/shows/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.paramountplus.com/movies/video/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.paramountplus.com/movies/paw-patrol-the-movie/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/',
         'only_matching': True,
     }, {
-        'url': 'https://www.paramountplus.com/movies/million-dollar-american-princesses-meghan-and-harry/C0LpgNwXYeB8txxycdWdR9TjxpJOsdCq',
+        # old format
+        'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/',
         'only_matching': True,
     }]
 

From 3361bc2191c9fdea98d5a83ba08a0dfcaf9b0bf7 Mon Sep 17 00:00:00 2001
From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com>
Date: Wed, 13 Apr 2022 07:21:23 -0500
Subject: [PATCH 184/394] Update options.py

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0a0641bd481..6521ad88162 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -270,11 +270,11 @@ def _comma_separated_values_options_callback(option, opt_str, value, parser):
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
-        help='Download only matching titles (regex or caseless sub-string)')
+        help='Download only matching titles (case-insensitive regex or sub-string)')
     selection.add_option(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
-        help='Skip download for matching titles (regex or caseless sub-string)')
+        help='Skip download for matching titles (case-insensitive regex or sub-string)')
     selection.add_option(
         '--max-downloads',
         dest='max_downloads', metavar='NUMBER', type=int, default=None,

From b3d2887ed81834e0c4afd4f98d6e770ca4f497ec Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Thu, 14 Apr 2022 17:11:14 +0500
Subject: [PATCH 185/394] fixed issues.

---
 youtube_dl/extractor/wikimedia.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index e5b6c32738d..27c023a6ec4 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,3 +1,5 @@
+# coding: utf-8
+from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import get_element_by_class, determine_ext, clean_html, KNOWN_EXTENSIONS
 import re
@@ -12,7 +14,7 @@ class WikimediaIE(InfoExtractor):
     _TEST = {
         'url': 'https://commons.wikimedia.org/wiki/File:Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
         'info_dict': {
-            'description': 'Deutsch:  Beschreibung auf der Seite: "Im Verlauf der Erdgeschichte glich das Klima einer Achterbahnfahrt. Die „Fieberkurve“ unseres Planeten zeigt die globalen Temperaturschwankungen bis heute – rekonstruiert anhand von historischen Klimadaten."\nZu Wikimedia Commons hochgeladen von: PantheraLeo1359531.\nHinweise zur Weiterverwendung: https://www.zdf.de/dokumentation/terra-x/terra-x-creative-commons-cc-100.html.\nVereinfachender Verlauf in der Geschichte der Erde, für die Zukunft spätestens ab dem Jahr 2050 mit spekulativem Verlauf in der Prognose (ausgeprägtes Global-warming-Szenario ist dargestellt).English:  Climate change, Temperature in history of Earth, Video of Terra X.',
+            'description': 'md5:D6F4C7BF1C0DB1EAE80371B1F93EA85E',
             'ext': 'webm',
             'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
             'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
@@ -56,12 +58,13 @@ def _real_extract(self, url):
         info['license'] = licenze
         info['author'] = author
 
-        subtitles = re.findall(r'\bsrc=\"\/w\/api\s*(.*?)\s*srt\b', str(webpage))
-        info['subtitles'] = {}
-        for sub in subtitles:
+        subtitles = {}
+        for sub in re.findall(r'\bsrc=\"\/w\/api\s*(.*?)\s*srt\b', str(webpage)):
             sub = 'https://commons.wikimedia.org/w/api' + sub + 'srt'
             lang = sub[sub.find('lang=') + 5:]
             lang = lang[:lang.find('&')]
             sub = sub.replace(';', '&')
             info['subtitles'][lang] = [{"ext": "srt", "url": sub}]
+
+        info['subtitles'] = subtitles
         return info

From ac260afbefe2c65166cb142c81b3ba47f8aecb7f Mon Sep 17 00:00:00 2001
From: Ahmed Alsuwaidi <aasuwaidi24@gmail.com>
Date: Fri, 15 Apr 2022 02:41:03 +0400
Subject: [PATCH 186/394] Updated streamgg

---
 youtube_dl/extractor/streamgg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamgg.py b/youtube_dl/extractor/streamgg.py
index a7ab85ca88f..5841973a6fe 100644
--- a/youtube_dl/extractor/streamgg.py
+++ b/youtube_dl/extractor/streamgg.py
@@ -5,7 +5,7 @@
 
 
 class StreamggIE(InfoExtractor):
-    _VALID_URL = r'https?://up.streamgg\.com/v/(?P<id>\w+)'
+    _VALID_URL = r'https?://(up.)?streamgg\.com/v/(?P<id>\w+)'
     _TEST = {
         "url": 'https://up.streamgg.com/v/r9gb3m95',
         'md5': 'eaf1f163635c868ecbba95d23ba83448',

From 328e6b7d47439bbe738dfce2d763ffee658ff9d3 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 15 Apr 2022 09:12:02 +0200
Subject: [PATCH 187/394] cleaning

---
 docs/supportedsites.md | 4 +++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8f5cc103eb0..8edbe71c770 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -805,9 +805,9 @@
  - **rtl2:you:series**
  - **RTP**
  - **RTS**: RTS.ch
- - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
+ - **rtve.es:play**: RTVE Play
  - **rtve.es:television**
  - **RTVNH**
  - **RTVS**
@@ -912,6 +912,8 @@
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreamFF**
+ - **Streamgg**
+ - **Streamwo**
  - **StreetVoice**
  - **StretchInternet**
  - **stv:player**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e19d1dfa1e8..5624e775964 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.12'
+__version__ = '2021.04.15'

From e8de1ac35b5b64152e6619dd7f5ed4cd55952cbf Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 15 Apr 2022 09:14:52 +0200
Subject: [PATCH 188/394] cleaning

---
 README.md              | 6 +++---
 docs/supportedsites.md | 7 +++++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 68ce7f6d897..87ed6b76d2e 100644
--- a/README.md
+++ b/README.md
@@ -141,10 +141,10 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          --playlist-items 1-3,7,10-13", it will
                                          download the videos at index 1, 2, 3,
                                          7, 10, 11, 12 and 13.
-    --match-title REGEX                  Download only matching titles (regex or
-                                         caseless sub-string)
+    --match-title REGEX                  Download only matching titles (case-
+                                         insensitive regex or sub-string)
     --reject-title REGEX                 Skip download for matching titles
-                                         (regex or caseless sub-string)
+                                         (case-insensitive regex or sub-string)
     --max-downloads NUMBER               Abort after downloading NUMBER files
     --min-filesize SIZE                  Do not download any videos smaller than
                                          SIZE (e.g. 50k or 44.6m)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8edbe71c770..99a1b7e9c3d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -584,6 +584,7 @@
  - **NBCSports**
  - **NBCSportsStream**
  - **NBCSportsVPlayer**
+ - **NBL**
  - **ndr**: NDR.de - Norddeutscher Rundfunk
  - **ndr:embed**
  - **ndr:embed:base**
@@ -696,6 +697,8 @@
  - **PalcoMP3:video**
  - **pandora.tv**: 판도라TV
  - **ParamountNetwork**
+ - **ParamountPlus**
+ - **ParamountPlusSeries**
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
@@ -1193,6 +1196,10 @@
  - **XTubeUser**: XTube user profile
  - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
+ - **XVideosChannel**
+ - **XVideosPlaylist**
+ - **XVideosRelated**
+ - **XVideosSearch**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **yahoo:gyao**

From f24ea24a29d101fc23c6ce61e0ef3d297ed35548 Mon Sep 17 00:00:00 2001
From: Chris W <84198105+L0wbyte@users.noreply.github.com>
Date: Sun, 17 Apr 2022 16:55:03 +0100
Subject: [PATCH 189/394] fix issues and dynamically fetch tokens

---
 youtube_dl/extractor/ciscolive.py | 59 ++++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dl/extractor/ciscolive.py
index da404e4dc57..470e42e9cbc 100644
--- a/youtube_dl/extractor/ciscolive.py
+++ b/youtube_dl/extractor/ciscolive.py
@@ -10,6 +10,7 @@
 )
 from ..utils import (
     clean_html,
+    ExtractorError,
     float_or_none,
     int_or_none,
     try_get,
@@ -21,12 +22,13 @@ class CiscoLiveBaseIE(InfoExtractor):
     # These appear to be constant across all Cisco Live presentations
     # and are not tied to any user session or event
     RAINFOCUS_API_URL = 'https://events.rainfocus.com/api/%s'
-    RAINFOCUS_API_PROFILE_ID = 'Na3vqYdAlJFSxhYTYQGuMbpafMqftalz'
-    RAINFOCUS_WIDGET_ID = 'n6l4Lo05R8fiy3RpUBm447dZN8uNWoye'
+    RAINFOCUS_API_PROFILE_ID = ''  # if blank will be fetched at runtime from site javascript
+    RAINFOCUS_WIDGET_ID = ''  # if blank will be fetched at runtime from site javascript
+    RAINFOCUS_TOKENS_URL = 'https://cdn-events.rainfocus.com/pages/cisco/clondemand/catalog.js'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5647924234001/SyK2FdqjM_default/index.html?videoId=%s'
 
     HEADERS = {
-        'Origin': 'https://ciscolive.cisco.com',
+        'Origin': 'https://ciscolive.com',
         'rfApiProfileId': RAINFOCUS_API_PROFILE_ID,
         'rfWidgetId': RAINFOCUS_WIDGET_ID,
     }
@@ -34,6 +36,21 @@ class CiscoLiveBaseIE(InfoExtractor):
     def _call_api(self, ep, rf_id, query, referrer, note=None):
         headers = self.HEADERS.copy()
         headers['Referer'] = referrer
+        if not self.RAINFOCUS_API_PROFILE_ID and not self.RAINFOCUS_WIDGET_ID:
+            rf_token_js = self._download_webpage(self.RAINFOCUS_TOKENS_URL, 'catalog.js', headers=headers)
+            if "apiToken" not in rf_token_js:
+                raise ExtractorError(
+                    'Unable to fetch api profile (apiToken) value.', expected=True)
+            if "widgetId" not in rf_token_js:
+                raise ExtractorError(
+                    'Unable to fetch widgetId value', expected=True)
+            api_token = self._html_search_regex(
+                r'(?:apiToken): \'(\w+)\',', rf_token_js, 'apiToken')
+            widget_id = self._html_search_regex(
+                r'(?:widgetId): \'(\w+)\',', rf_token_js, 'widgetId')
+            if api_token and widget_id:
+                headers['rfApiProfileId'] = api_token
+                headers['rfWidgetId'] = widget_id
         return self._download_json(
             self.RAINFOCUS_API_URL % ep, rf_id, note=note,
             data=urlencode_postdata(query), headers=headers)
@@ -65,19 +82,18 @@ def _parse_rf_item(self, rf_item):
 
 
 class CiscoLiveSessionIE(CiscoLiveBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/[^#]*#/session/(?P<id>[^/?&]+)'
+    _VALID_URL = r'https?://(?:www\.)?ciscolive\.com/[^#]*#/session/(?P<id>[^/?&]+)'
     _TESTS = [{
-        'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
-        'md5': 'c98acf395ed9c9f766941c70f5352e22',
+        'url': 'https://www.ciscolive.com/on-demand/on-demand-library.html?search=#/session/16360600004400017rMx',
+        'md5': 'e0f5b0b2927b586ebff619294fec6926',
         'info_dict': {
-            'id': '5803694304001',
+            'id': '6128601216001',
             'ext': 'mp4',
-            'title': '13 Smart Automations to Monitor Your Cisco IOS Network',
-            'description': 'md5:ec4a436019e09a918dec17714803f7cc',
-            'timestamp': 1530305395,
-            'upload_date': '20180629',
+            'title': 'A Deeper Dive into the Telco Cloud Architecture Evolution to support 5G and MEC - BRKSPG-1565',
+            'description': 'md5:04bc54ec8ede2bbd9d21264bfaf16cb3',
+            'timestamp': 1580474594,
+            'upload_date': '20200131',
             'uploader_id': '5647924234001',
-            'location': '16B Mezz.',
         },
     }, {
         'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.event=ciscoliveemea2019#/session/15361595531500013WOU',
@@ -85,25 +101,30 @@ class CiscoLiveSessionIE(CiscoLiveBaseIE):
     }, {
         'url': 'https://www.ciscolive.com/global/on-demand-library.html?#/session/1490051371645001kNaS',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ciscolive.com/on-demand/on-demand-library.html?#/session/16360600774720017Iby',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         rf_id = self._match_id(url)
         rf_result = self._call_api('session', rf_id, {'id': rf_id}, url)
+        if int(rf_result['responseCode']) != 0:
+            raise ExtractorError(
+                'Rainfocus session api returned a non success responseCode: %s. api keys might be invalid' % rf_result['responseCode'], expected=True)
         return self._parse_rf_item(rf_result['items'][0])
 
 
 class CiscoLiveSearchIE(CiscoLiveBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/(?:global/)?on-demand-library(?:\.html|/)'
+    _VALID_URL = r'https?://(?:www\.)?ciscolive\.com/(?:global|on-demand/)?on-demand-library(?:\.html|/)'
     _TESTS = [{
-        'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
+        'url': 'https://www.ciscolive.com/on-demand/on-demand-library.html?search.event=1636046385176005FbBU&search.technology=scpsTechnology_dataCenter&search.technicallevel=scpsSkillLevel_aintroductory#/',
         'info_dict': {
             'title': 'Search query',
         },
-        'playlist_count': 5,
+        'playlist_count': 3,  # example returns 4 results, only 3 have video
     }, {
-        'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
-        'only_matching': True,
+        'url': 'https://www.ciscolive.com/on-demand/on-demand-library.html?search.technology=scpsTechnology_automation&search.technicallevel=scpsSkillLevel_cadvanced#/',
     }, {
         'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.technicallevel=scpsSkillLevel_aintroductory&search.event=ciscoliveemea2019&search.technology=scpsTechnology_dataCenter&search.focus=scpsSessionFocus_bestPractices#/',
         'only_matching': True,
@@ -124,6 +145,10 @@ def _entries(self, query, url):
             results = self._call_api(
                 'search', None, query, url,
                 'Downloading search JSON page %d' % page_num)
+            if int(results['totalSearchItems']) == 0:
+                raise ExtractorError(
+                    'Search api returned 0 items (if matches are expected rfApiProfileId may be invalid)',
+                    expected=True)
             sl = try_get(results, lambda x: x['sectionList'][0], dict)
             if sl:
                 results = sl

From 272bfb3e477335b7ef116db9b0546e291e1800e4 Mon Sep 17 00:00:00 2001
From: cens0r <74016750+cens0r@users.noreply.github.com>
Date: Mon, 18 Apr 2022 19:36:29 +0200
Subject: [PATCH 190/394] Add CamWhoresBay (video / all model videos)

---
 youtube_dl/extractor/camwhoresbay.py | 143 +++++++++++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 youtube_dl/extractor/camwhoresbay.py

diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py
new file mode 100644
index 00000000000..c158a7bfd20
--- /dev/null
+++ b/youtube_dl/extractor/camwhoresbay.py
@@ -0,0 +1,143 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import re
+import itertools
+from ..utils import (
+    js_to_json,
+)
+
+
+class CamWhoresBayVideoIE(InfoExtractor):
+    IE_NAME = 'camwhoresbay:video'
+    _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/videos/(?P<id>[0-9]+)/.+'
+    _TEST = {
+        'url': 'https://www.camwhoresbay.com/videos/484472/kirsten-xxx-2022-01-19/',
+        'md5': 'ae3c5c34866afbd062adf5d2321d66f3',
+        'info_dict': {
+            'id': '484472',
+            'ext': 'mp4',
+            'title': 'Kirsten_Xxx 2022-01-19',
+            'uploader': '789sani',
+            'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
+            # 'formats': [{
+            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472\.mp4/\?rnd=.+',
+            #     'height': 360
+            # }, {
+            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_480p\.mp4/\?rnd=.+',
+            #     'height': 480
+            # }, {
+            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
+            #     'height': 720
+            # }]
+            'thumbnail': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg',
+            'thumbnails': [{
+                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg',
+                'height': 360
+            }, {
+                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_480p.mp4.jpg',
+                'height': 480
+            }, {
+                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg',
+                'height': 720
+            }],
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    @staticmethod
+    def _parse_formats(flashvars):
+        formats = []
+        rnd = flashvars.get('rnd')
+        for k, v in flashvars.items():
+            if re.match(r"^video_(alt_)?url\d?$", k):
+                f = {
+                    'url': '{}?rnd={}'.format(v, rnd),
+                    'height': int(flashvars['{}_text'.format(k)][:-1])
+                }
+                formats.append(f)
+        return formats
+
+    @staticmethod
+    def _parse_thumbnails(flashvars):
+        thumbnails = []
+        for k, v in flashvars.items():
+            if re.match(r"^preview_url\d$", k):
+                t = {
+                    'url': 'https:{}'.format(v),
+                    'height': int(flashvars[k.replace('_url', '_height')])
+                }
+                thumbnails.append(t)
+        return thumbnails
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<div class=\"headline\">\n\s*<h1>(.+?)</h1>', webpage, 'title')
+
+        flashvars = self._parse_json(
+            self._search_regex(r'var\s+flashvars\s+=\s+({\n.+});', webpage, 'flashvars', default='{}'),
+            video_id, transform_source=js_to_json)
+        uploader = self._search_regex(r'class=\"avatar\" href=\"http.+/\" title=\"(.+)\"', webpage, 'uploader',
+                                      fatal=False)
+
+        formats = self._parse_formats(flashvars)
+        thumbnails = self._parse_thumbnails(flashvars)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'url': formats[0]['url'],
+            'formats': formats,
+            'thumbnail': thumbnails[0]['url'],
+            'thumbnails': thumbnails,
+        }
+
+
+class CamWhoresBayModelIE(CamWhoresBayVideoIE):
+    IE_NAME = 'camwhoresbay:model'
+    _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/models/(?P<id>[0-9a-z\-]+)/?'
+    _MORE_PAGES_INDICATOR = r'<li\s+class="next"><a\s+href="#videos"'
+    _TITLE = None
+    _TITLE_RE = r'<div\s+class=\"headline\">\n\s*<h1>\s*(.*?)\'s New Videos.+</h1>'
+    _BASE_URL_TEMPL = 'https://www.camwhoresbay.com/models/%s/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from='
+    _TEST = {
+        'url': 'https://www.camwhoresbay.com/models/kirsten-xxx/',
+        'info_dict': {
+            'id': 'kirsten-xxx',
+            'title': 'Kirsten_xxx',
+        }
+    }
+    _PAGE_SIZE = 35
+
+    def _extract_list_title(self, webpage):
+        return self._TITLE or self._html_search_regex(
+            self._TITLE_RE, webpage, 'list title', fatal=False)
+
+    def _title_and_entries(self, model_id, base_url):
+        for page in itertools.count(1):
+            webpage = self._download_webpage('%s%d' % (base_url, page), model_id, 'Downloading page %s' % page)
+
+            if page == 1:
+                yield self._extract_list_title(webpage)
+
+            for video_id in re.findall(r'<div\s+class=\"video-item\s*\">\n\s*<a href=\"(.*?)\"\s+title', webpage):
+                yield self.url_result(video_id)
+
+            if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
+                break
+
+    def _extract_videos(self, list_id, base_url):
+        title_and_entries = self._title_and_entries(list_id, base_url)
+        list_title = next(title_and_entries)
+        return self.playlist_result(title_and_entries, list_id, list_title)
+
+    def _real_extract(self, url):
+        model_id = self._match_id(url)
+        return self._extract_videos(model_id, self._BASE_URL_TEMPL % model_id)

From 771bbcb453f047198bce0d7c0b1b950d45965b1e Mon Sep 17 00:00:00 2001
From: cens0r <74016750+cens0r@users.noreply.github.com>
Date: Mon, 18 Apr 2022 19:37:41 +0200
Subject: [PATCH 191/394] Enable CamWhoresBay extractor

---
 youtube_dl/extractor/extractors.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..e64d09cd118 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -164,6 +164,10 @@
 )
 from .cammodels import CamModelsIE
 from .camtube import CamTubeIE
+from .camwhoresbay import (
+    CamWhoresBayVideoIE,
+    CamWhoresBayModelIE,
+)
 from .camwithher import CamWithHerIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE

From 31825563c7cf9a12a13446d35a2df884eaa499ba Mon Sep 17 00:00:00 2001
From: cens0r <74016750+cens0r@users.noreply.github.com>
Date: Mon, 18 Apr 2022 19:50:27 +0200
Subject: [PATCH 192/394] minor fixes

---
 youtube_dl/extractor/camwhoresbay.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py
index c158a7bfd20..2c00e4b2e08 100644
--- a/youtube_dl/extractor/camwhoresbay.py
+++ b/youtube_dl/extractor/camwhoresbay.py
@@ -31,7 +31,6 @@ class CamWhoresBayVideoIE(InfoExtractor):
             #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
             #     'height': 720
             # }]
-            'thumbnail': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg',
             'thumbnails': [{
                 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg',
                 'height': 360
@@ -93,9 +92,8 @@ def _real_extract(self, url):
             'id': video_id,
             'title': title,
             'uploader': uploader,
-            'url': formats[0]['url'],
+            'url': formats[0].get('url'),
             'formats': formats,
-            'thumbnail': thumbnails[0]['url'],
             'thumbnails': thumbnails,
         }
 

From e84b4d1c9ea50e11b687df93f3407bfabba9b729 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 19 Apr 2022 04:05:23 +0100
Subject: [PATCH 193/394] Sanitise info HTML for yt-dlp's
 get_elements_by_class()

---
 youtube_dl/extractor/porn91.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index b299c7b9ad3..340903f450b 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -39,6 +39,7 @@ class Porn91IE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        # set language for page to be extracted
         self._set_cookie('91porn.com', 'language', 'cn_CN')
 
         webpage = self._download_webpage(
@@ -57,7 +58,9 @@ def _real_extract(self, url):
 
         info_dict = self._parse_html5_media_entries(url, '<video>%s</video>' % (video_link_url, ), video_id)[0]
 
+        # extract various fields in <tag class=info>Name: value</tag>
         FIELD_MAP = {
+            # cn_CN name: (yt-dl key, value parser, en name)
             '时长': ('duration', parse_duration, 'Runtime', ),
             '查看': ('view_count', str_to_int, 'Views', ),
             '留言': ('comment_count', str_to_int, 'Comments', ),
@@ -67,7 +70,14 @@ def _real_extract(self, url):
             '__ignore__': ('description', strip_or_none, 'Description', ),
             '作者': ('uploader', strip_or_none, 'From', ),
         }
-        for elt in get_elements_by_class('info', re.sub(r'</span>\s*<span\b[^>]*?>', '', webpage)) or []:
+        # yt-dl's original implementation of get_elements_by_class() uses regex
+        # yt-dlp uses an actual HTML parser, and can be confused by bad HTML fragments
+        for elt in get_elements_by_class(
+                'info',
+                # concatenate <span>s ...
+                re.sub(r'(?i)</span>\s*<span\b[^>]*?>', '',
+                       # ... and strip out possibly unbalanced <font> for yt-dlp
+                       re.sub(r'(?i)(?:<font\b[^>]*?>|</font\s*>)', '', webpage))) or []:
             elt = re.split(r':\s*', clean_html(elt), 1)
             if len(elt) != 2 or elt[1] == '':
                 continue

From c555e5e478b3d9bc5861bcbd4a9cfe5d61958758 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 19 Apr 2022 12:57:22 +0100
Subject: [PATCH 194/394] [Facebook] Check for redirection to URL with `login`,
 or not a permalink, or not including the ID

---
 youtube_dl/extractor/facebook.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index a3f1bd32555..389d1f7bf91 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -410,8 +410,15 @@ def _real_initialize(self):
         self._login()
 
     def _extract_from_url(self, url, video_id):
-        webpage = self._download_webpage(
+        webpage, urlh = self._download_webpage_handle(
             url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
+        login_or_id_check = re.search(r'(?P<login>\blogin\b)|/permalink/|[/=?]' + video_id, urlh.geturl())
+        if login_or_id_check and login_or_id_check.group('login'):
+            self.raise_login_required()
+        elif not login_or_id_check:
+            raise ExtractorError(
+                'Facebook redirected to a generic page (%s): the video may be unavailable or you may need to log in.' % urlh.geturl(),
+                expected=True)
 
         def extract_metadata(webpage):
             video_title = self._html_search_regex(

From f2759dea09b2ab9d4635e71fc73ef7b5f6803b51 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Wed, 20 Apr 2022 09:17:19 +0200
Subject: [PATCH 195/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index ae74b03f401..514d69a2a92 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -38,18 +38,20 @@ def _real_extract(self, url):
         if not kvs_version.startswith("5."):
             self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")
 
-        title = self._html_search_regex(r'<title>(?:Video: )?(.+?)(?: - ThisVid(?:.com| tube))?</title>', webpage, 'title')
+        title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
         # video_id, video_url and license_code from the 'flashvars' JSON object:
-        video_id = self._html_search_regex(r"video_id: '([0-9]+)',", webpage, 'video_id')
-        video_url = self._html_search_regex(r"video_url: '(function/0/.+?)',", webpage, 'video_url')
-        license_code = self._html_search_regex(r"license_code: '([0-9$]{16})',", webpage, 'license_code')
-        thumbnail = self._html_search_regex(r"preview_url: '((?:https?:)?//media.thisvid.com/.+?.jpg)',", webpage, 'thumbnail', fatal=False)
+        video_id = self._html_search_regex(r"video_id:\s+'([0-9]+)',", webpage, 'video_id')
+        video_url = self._html_search_regex(r"video_url:\s+'(function/0/.+?)',", webpage, 'video_url')
+        license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')
+        thumbnail = self._html_search_regex(r"preview_url:\s+'((?:https?:)?//media\.thisvid\.com/.+?\.jpg)',", webpage, 'thumbnail', fatal=False)
+        uploader = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/[0-9]+/">(.+?)</a>', webpage, 'uploader')
+        uploader_id = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/([0-9]+)/">.+?</a>', webpage, 'uploader_id')
         if thumbnail.startswith("//"):
             thumbnail = "https:" + thumbnail
         if (re.match(self._VALID_URL, url).group('type') == "videos"):
             display_id = main_id
         else:
-            display_id = self._search_regex(r'<link rel="canonical" href="' + self._VALID_URL + r'">', webpage, 'display_id', fatal=False),
+           display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False)
 
         return {
             'id': video_id,

From d2d29813fd3cd020c20bb6f303364dbe6a88669c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Dobo=C5=A1?= <jakub.dobos@gab.sk>
Date: Wed, 20 Apr 2022 10:40:39 +0200
Subject: [PATCH 196/394] fix the fix

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0b2bec38369..685e0c5d2dd 100644
--- a/README.md
+++ b/README.md
@@ -1406,7 +1406,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.libera.chat/#youtube-dl) on freenode ([webchat](https://web.libera.chat/?randomnick=1&channels=youtube-dl)).
+Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.libera.chat/#youtube-dl) on libera.chat ([webchat](https://web.libera.chat/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```

From 8934a232fb1e2491932fb627df3b4329f8318317 Mon Sep 17 00:00:00 2001
From: Oliver Pfeiffer <olipfei@netscape.net>
Date: Wed, 20 Apr 2022 23:05:08 +0200
Subject: [PATCH 197/394] [douyin] Incorporated dirkf's remarks part 2.
 Adjusted test data.

---
 youtube_dl/extractor/douyin.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/douyin.py b/youtube_dl/extractor/douyin.py
index 082f7e850bc..1ec49ce8e2e 100644
--- a/youtube_dl/extractor/douyin.py
+++ b/youtube_dl/extractor/douyin.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -14,12 +15,12 @@ class DouyinVideoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://www.douyin.com/video/6989098563519270181',
-        'md5': 'b4c99f3be1be343948a9eaaea8303d27',
+        'md5': '99c5667992b8a8d46c145907f677c92b',
         'info_dict': {
             'id': '6989098563519270181',
-            'url': r're:^https://v3-dy-o.zjcdn.com/[0-9a-f]{32}/[0-9a-f]{8}/video/tos/cn/tos-cn-ve-15/[0-9a-f]{32}/?.*',
+            'url': 'https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0300fg10000c3v47dbc77u9fvb20tbg&ratio=720p&line=0',
             'title': '杨集#我的家乡 ',
-            'uploader': '永恒的爱',
+            'uploader': '🌹永恒的爱🌹',
             'uploader_id': '104081949894',
             'timestamp': 1627276320000.0,
             'ext': 'mp4'
@@ -45,13 +46,13 @@ def _real_extract(self, url):
             'id': video_id,
             'title': item['desc'],
             'url': item['video']['play_addr']['url_list'][0],
-            'uploader': try_get(item, lambda x: x['author']['nickname']),
-            'uploader_id': try_get(item, lambda x: x['author']['uid']),
+            'uploader': try_get(item, lambda x: x['author']['nickname'], compat_str),
+            'uploader_id': try_get(item, lambda x: x['author']['uid'], compat_str),
             'duration': int_or_none(item.get('duration') or try_get(item, lambda x: x['video']['duration'], int), scale=1000),
             'timestamp': int_or_none(item.get('create_time'), invscale=1000),
-            'width': try_get(item, lambda x: x['video']['width']),
-            'height': try_get(item, lambda x: x['video']['height']),
-            'vbr': try_get(item, lambda x: x['video']['bit_rate']),
+            'width': try_get(item, lambda x: x['video']['width'], int),
+            'height': try_get(item, lambda x: x['video']['height'], int),
+            'vbr': try_get(item, lambda x: x['video']['bit_rate'], int),
             'ext': 'mp4'
         }
 
@@ -84,12 +85,13 @@ def _real_extract(self, url):
             if aweme_list is None:
                 raise ExtractorError('JSON response does not contain aweme_list', video_id=sec_uid)
 
-            entries.extend([self.url_result('https://www.douyin.com/video/%s' % aweme['aweme_id'],
-                                            ie=DouyinVideoIE.ie_key(), video_id=aweme['aweme_id'])
-                            for aweme in aweme_list])
+            entries.extend([self.url_result('https://www.douyin.com/video/%s' % aweme_id,
+                                            ie=DouyinVideoIE.ie_key(), video_id=aweme_id)
+                            for aweme_id in filter(None,
+                                                   (aweme.get('aweme_id') for aweme in aweme_list
+                                                    if isinstance(aweme, dict)))])
 
-            has_more = post['has_more']
-            max_cursor = post['max_cursor']
+            has_more = post.get('has_more')
+            max_cursor = post.get('max_cursor')
 
         return self.playlist_result(orderedSet(entries), sec_uid)
-

From 26b00c422a36cab1a436f92a60e7043a7349b922 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 23 Apr 2022 10:12:12 +0200
Subject: [PATCH 198/394] cleaning

---
 docs/supportedsites.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 99a1b7e9c3d..f0be2b04dea 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -248,6 +248,8 @@
  - **dlive:stream**
  - **dlive:vod**
  - **Dotsub**
+ - **DouyinUser**
+ - **DouyinVideo**
  - **DouyuShow**
  - **DouyuTV**: 斗鱼
  - **DPlay**

From 2f65e205e18c42fe5f83d1e7a221ae469d463cd4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 25 Apr 2022 15:24:32 +0100
Subject: [PATCH 199/394] Fix TT blocking

Add TikTokVM
Partial fix for TikTokUser
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/tiktok.py     | 163 +++++++++++++++++++----------
 2 files changed, 107 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6e8fc3961b7..34c374a0fcf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1249,6 +1249,7 @@
 from .tiktok import (
     TikTokIE,
     TikTokUserIE,
+    TikTokVMIE,
 )
 from .tinypic import TinyPicIE
 from .tmz import (
diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index 99a0c07d7e0..49df0844b85 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -1,15 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from .generic import GenericIE
 from ..compat import (
-    compat_str,
-    compat_urllib_request,
+    compat_kwargs,
 )
 from ..utils import (
+    dict_get,
     ExtractorError,
     float_or_none,
-    HEADRequest,
+    get_element_by_id,
     int_or_none,
     str_or_none,
     try_get,
@@ -17,24 +20,37 @@
 )
 
 
+# decorator enforces UA that TT doesn't block
+def vanilla_UA_request(func):
+
+    vanilla_UA = 'Mozilla/5.0'
+
+    def wrapped(*args, **kwargs):
+        headers = kwargs.get('headers', {})
+        if 'User-Agent' not in headers:
+            headers['User-Agent'] = vanilla_UA
+            kwargs.update({'headers': headers, })
+            kwargs = compat_kwargs(kwargs)
+        return func(*args, **kwargs)
+
+    return wrapped
+
+
 class TikTokBaseIE(InfoExtractor):
-    def _download_webpage(
-            self, url_or_request, video_id, note=None, errnote=None,
-            fatal=True, tries=1, timeout=5, encoding=None, data=None,
-            headers={}, query={}, expected_status=None, setup=True):
-
-        if setup:
-            url = url_or_request.geturl() if isinstance(url_or_request, compat_urllib_request.Request) else url_or_request
-            # dummy request to set cookies
-            self._request_webpage(
-                HEADRequest(url), video_id,
-                note=False, errnote='Could not send HEAD request to %s' % url,
-                fatal=False, headers=headers)
-
-        return super(TikTokBaseIE, self)._download_webpage(
-            url_or_request, video_id, note=note, errnote=errnote,
-            fatal=fatal, tries=tries, timeout=timeout, encoding=encoding, data=data,
-            headers=headers, query=query, expected_status=expected_status)
+    IE_DESC = 'Abstract base for TikTok extractors'
+    IE_NAME = 'tiktok:base'
+
+    @vanilla_UA_request
+    def _request_webpage(self, *args, **kwargs):
+        return super(TikTokBaseIE, self)._request_webpage(*args, **kwargs)
+
+    def _get_SIGI_STATE(self, video_id, html):
+        state = self._parse_json(
+            get_element_by_id('SIGI_STATE', html)
+            or self._search_regex(
+                r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
+                html, 'sigi data', default='{}', group='json'), video_id)
+        return state if isinstance(state, dict) else {}
 
     def _extract_video(self, data, video_id=None):
         video = try_get(data, lambda x: x['video'], dict)
@@ -102,18 +118,20 @@ def _extract_video(self, data, video_id=None):
 
 
 class TikTokIE(TikTokBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@[^/]+/video/(?P<id>\d+)'
+    IE_DESC = 'TikTok video extractor'
+    IE_NAME = 'tiktok'
+    _VALID_URL = r'(?:https?://(?:(?:www|m)\.)?tiktok\.com/@[^/]+/video/|tiktok:(?P<user_id>[^/?#&]+):)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.tiktok.com/@zureeal/video/6606727368545406213',
         'md5': '163ceff303bb52de60e6887fe399e6cd',
         'info_dict': {
             'id': '6606727368545406213',
             'ext': 'mp4',
-            'title': 'md5:24acc456b62b938a7e2dd88e978b20d9',
+            'title': 'md5:363e08ccb6c691314710429f379bffe5',
             'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
             'thumbnail': r're:^https?://.*',
             'duration': 15,
-            'uploader': 'md5:24acc456b62b938a7e2dd88e978b20d9',
+            'uploader': 'md5:363e08ccb6c691314710429f379bffe5',
             'uploader_id': '188294915489964032',
             'timestamp': 1538248586,
             'upload_date': '20180929',
@@ -124,78 +142,85 @@ class TikTokIE(TikTokBaseIE):
         }
     }]
 
+    def _real_initialize(self):
+        # Setup session (will set necessary cookies)
+        self._request_webpage(
+            'https://www.tiktok.com/', None, note='Setting up session')
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        m = re.match(self._VALID_URL, url).groupdict()
+        video_id = m['id']
+        if 'user_id' in m:
+            url = 'https://www.tiktok.com/@%(user_id)s/video/%(id)s/' % m
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url.replace('://m.', '://www.'), video_id)
 
-        page_props = self._parse_json(self._search_regex(
-            r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
-            webpage, 'sigi data', default='{}', group='json'), video_id)
-        data = try_get(page_props, lambda x: x['ItemModule'][video_id]['video'], dict)
-        if data:
+        page_props = self._get_SIGI_STATE(video_id, webpage)
+        if try_get(page_props, lambda x: x['ItemModule'][video_id]['video'], dict):
             data = page_props['ItemModule'][video_id]
             if data.get('privateItem'):
                 raise ExtractorError('This video is private', expected=True)
             return self._extract_video(data, video_id)
 
-        page_props = self._parse_json(self._search_regex(
-            r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
-            webpage, 'data'), video_id)['props']['pageProps']
+        # old page format - may still be served?
+        page_props = self._parse_json(
+            get_element_by_id('__NEXT_DATA__', webpage),
+            video_id)['props']['pageProps']
         data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
         if not data and page_props.get('statusCode') == 10216:
             raise ExtractorError('This video is private', expected=True)
         return self._extract_video(data, video_id)
 
 
-class TikTokUserIE(TikTokBaseIE):
-    _VALID_URL = r'https://(?:www\.)?tiktok\.com/@(?P<id>[^/?#&]+)'
+class TikTokUserIE(TikTokIE):
+    IE_DESC = 'TikTok user profile extractor'
+    IE_NAME = 'tiktok:user'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?tiktok\.com/@(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.tiktok.com/@zureeal',
         'info_dict': {
             'id': '188294915489964032',
         },
-        'playlist_mincount': 24,
+        'playlist_mincount': 30,
+        'expected_warnings': [
+            'More videos are available',
+        ],
     }]
 
     @classmethod
     def suitable(cls, url):
-        return False if TikTokIE.suitable(url) else super(TikTokUserIE, cls).suitable(url)
+        return False if TikTokIE.suitable(url) else super(TikTokBaseIE, cls).suitable(url)
 
     def _real_extract(self, url):
         user_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, user_id)
-        page_props = self._parse_json(self._search_regex(
-            r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
-            webpage, 'sigi data', default='{}', group='json'), user_id)
+        webpage = self._download_webpage(url.replace('://m.', '://www.'), user_id)
+
+        page_props = self._get_SIGI_STATE(user_id, webpage)
         user_data = try_get(page_props, lambda x: x['UserModule']['users'], dict)
-        entries = []
         if user_data:
             num_id = try_get(
                 user_data.values(),
-                lambda x: [user['id'] for user in x if user['uniqueId'] == user_id][0],
-                compat_str)
-            item_data = try_get(page_props, lambda x: x['ItemModule'], dict)
-            if item_data:
-                item_data = item_data.values()
-            for data in item_data or []:
-                if data.get('privateItem'):
-                    continue
-                item = self._extract_video(data, user_id)
-                if item:
-                    entries.append(item)
-            result = entries and self.playlist_result(entries, num_id)
-            if not result:
-                item_data = try_get(page_props, lambda x: x['ItemList']['user-post']['list'], list)
-                result = self.playlist_from_matches(item_data, playlist_id=num_id, getter=lambda m: 'tiktok:%s' % (m, ))
+                lambda x: [user['id'] for user in x if user['uniqueId'] == user_id and user['id'].isdigit()][0])
+            item_data = try_get(page_props, lambda x: x['ItemList']['user-post'], dict) or {}
+            item_list = dict_get(item_data, ('list', 'browserList'))
+            if not item_list:
+                item_list = try_get(item_data, lambda x: [y['id'] for y in x['preloadList']], list)
+            entries = ['tiktok:%s:%s' % (user_id, x, ) for x in item_list or []]
+            if item_data.get('hasMore', False):
+                self._downloader.report_warning('More videos are available but the current extractor doesn\'t know how to find them')
+
+            result = entries and self.playlist_from_matches(entries, num_id, ie='TikTok')
             if result:
                 result['display_id'] = user_id
                 return result
 
+        """
+        # this no longer works 2022-01
         data = self._download_json(
             'https://m.tiktok.com/h5/share/usr/list/%s/' % user_id, user_id,
             query={'_signature': '_'})
+        entries = []
         for aweme in data['aweme_list']:
             try:
                 entry = self._extract_video(aweme)
@@ -205,3 +230,27 @@ def _real_extract(self, url):
             entries.append(entry)
 
         return self.playlist_result(entries, user_id)
+        """
+
+
+class TikTokVMIE(GenericIE):
+    IE_DESC = 'Resolver for TikTok shortcuts'
+    IE_NAME = 'tiktok:shortcut'
+    _VALID_URL = r'https?://(?:vm\.tiktok\.com|m\.tiktok\.com/v)/(?P<id>[^/?#&.]+)'
+    _TESTS = [{
+        'url': 'https://vm.tiktok.com/ZMLesneqK/',
+        'info_dict': {
+            'id': '7054218882072055046',
+            'ext': 'mp4',
+            'title': 'EddY',
+            'upload_date': '20220117',
+            'description': 'Hilft bestimmt gegen nervige Anrufer! 😂 #telefon #call #prank #fail #sprecher #stimme #voice #band #ansage #sound #comedy #unterhaltung #scammer #fy',
+            'timestamp': 1642438324,
+            'uploader': 'EddY',
+            'uploader_id': '6850021004246467590',
+        },
+    }]
+
+    @vanilla_UA_request
+    def _request_webpage(self, *args, **kwargs):
+        return super(GenericIE, self)._request_webpage(*args, **kwargs)

From 876d9b8f49c23aa5df79d451cea52e7a909c770d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 28 Apr 2022 01:17:43 +0100
Subject: [PATCH 200/394] [TalkTV] Add extractors for TalkTV (UK) shows and
 series

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/talktv.py     | 176 +++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+)
 create mode 100644 youtube_dl/extractor/talktv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a0c..0effe98c0cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1214,6 +1214,10 @@
     TagesschauPlayerIE,
     TagesschauIE,
 )
+from .talktv import (
+    TalkTVIE,
+    TalkTVSeriesIE,
+)
 from .tass import TassIE
 from .tbs import TBSIE
 from .tdslifeway import TDSLifewayIE
diff --git a/youtube_dl/extractor/talktv.py b/youtube_dl/extractor/talktv.py
new file mode 100644
index 00000000000..75b6acc78cf
--- /dev/null
+++ b/youtube_dl/extractor/talktv.py
@@ -0,0 +1,176 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import calendar
+from datetime import datetime
+import time
+
+from .common import InfoExtractor
+
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    determine_ext,
+    extract_attributes,
+    ExtractorError,
+    get_elements_by_class,
+    HEADRequest,
+    parse_duration,
+    try_get,
+    unified_timestamp,
+    url_or_none,
+    urljoin,
+)
+
+
+class TalkTVIE(InfoExtractor):
+    IE_NAME = 'talk.tv'
+    _VALID_URL = r'https?://watch\.talk\.tv/(?P<id>watch/(?:vod|replay)/\d+|live)'
+    _TESTS = [{
+        'url': 'https://watch.talk.tv/watch/replay/12216792',
+        'md5': 'dc9071f7d26f48ce4057a98425894eb3',
+        'info_dict': {
+            'id': '12216792',
+            'ext': 'mp4',
+            'title': 'Piers Morgan Uncensored',
+            'description': 'The host interviews former US President Donald Trump',
+            'timestamp': 1650917390,
+            'upload_date': '20220425',
+            'duration': float,
+        },
+        'params': {
+            'skip_download': True,  # adaptive download
+        },
+    }, {
+        'url': 'https://watch.talk.tv/live',
+        'info_dict': {
+            'id': 'live',
+            'ext': 'mp4',
+            'title': 'Piers Morgan Uncensored',
+            'description': compat_str,
+            'timestamp': int,
+            # needs core fix to force compat_str type
+            'upload_date': r're:\d{8}',
+            'duration': float,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url).rsplit('/', 1)[-1]
+        is_live = (video_id == 'live')
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'(?is)<h1\b[^>]+>\s*(.+?)\s*</h1', webpage, 'title')
+        player = self._search_regex(r'''(<[dD][iI][vV]\b[^>]+?\bid\s*=\s*(?P<q>"|')player(?P=q)[^>]*>)''', webpage, video_id)
+        player = extract_attributes(player)
+        expiry = player.get('expiry')
+        if expiry is not None and expiry < time.time():
+            raise ExtractorError('Video has expired', expected=True)
+        api_info = self._download_json(
+            'https://mm-v2.simplestream.com/ssmp/api.php?id=%(data-id)s&env=%(data-env)s' % player,
+            video_id, note='Downloading API info', fatal=False)
+        player['api_url'] = (
+            url_or_none(try_get(api_info, lambda x: x['response']['api_hostname']))
+            or 'https://v2-streams-elb.simplestreamcdn.com')
+        headers = {'Referer': url, }
+        for item in ('uvid', 'token', ('expiry', 'Token-Expiry')):
+            if isinstance(item, compat_str):
+                name = item.capitalize()
+            else:
+                item, name = item
+            val = player.get('data-' + item)
+            if val is not None:
+                headers[name] = val
+
+        stream_info = self._download_json(
+            '%(api_url)s/api/%(data-type)s/stream/%(data-uvid)s?key=%(data-key)s&platform=firefox&cc=%(data-country)s' % player,
+            video_id, headers=headers)
+        error = try_get(stream_info, lambda x: x['response']['error'])
+        if error:
+            raise ExtractorError('Streaming API reported: ' + error, expected=True)
+        fmt_url = (stream_info['response'].get('drm') in (None, False)) and stream_info['response']['stream']
+        formats = []
+        duration = None
+        description = None
+        timestamp = None
+        if fmt_url:
+            ext = determine_ext(fmt_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    fmt_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', live=is_live, fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    fmt_url, video_id, mpd_id='dash', live=is_live, fatal=False))
+            else:
+                formats.append({
+                    'url': fmt_url,
+                })
+            if not is_live:
+                res = self._request_webpage(HEADRequest(fmt_url), video_id, note='Checking date', fatal=False)
+                if res is not False:
+                    timestamp = unified_timestamp(res.info().getheader('last-modified'))
+
+        self._sort_formats(formats)
+
+        text_fields = get_elements_by_class('text-start', webpage)
+        for text in text_fields:
+            text = clean_html(text)
+            if text.startswith('EPISODE'):
+                duration = parse_duration(
+                    self._html_search_regex(r'^EPISODE\b\W*(\w[\w\s]*?)\s*$', text, 'duration', default=None))
+            elif text.startswith('Live'):
+                duration = self._html_search_regex(r'^Live\b(?:<[^>]+>|\W)*([0-2]?\d:\d{2}\s*-\s*[0-2]?\d:\d{2})\s*$', text, 'duration', default=None)
+                duration = list(map(lambda x: datetime.strptime(x, '%H:%M'), re.split(r'\s*-\s*', duration)))
+                if None not in duration and len(duration) == 2:
+                    timestamp = datetime.now().replace(hour=duration[0].hour, minute=duration[0].minute, second=0, microsecond=0)
+                    timestamp = calendar.timegm(timestamp.timetuple())
+                    duration = duration[1] - duration[0]
+                    try:
+                        duration = duration.total_seconds()
+                    except AttributeError:
+                        # Py 2.6
+                        duration = duration.td_seconds
+                    if duration is not None and duration < 0:
+                        duration += 24 * 3600
+            else:
+                description = text
+
+        return {
+            # ensure live has a fixed ID
+            'id': player['data-uvid'] if not is_live else video_id,
+            'title': title,
+            'display_id': video_id if not is_live else player['data-uvid'],
+            'formats': formats,
+            'thumbnail': player.get('data-poster'),
+            'duration': duration,
+            'timestamp': timestamp,
+            'description': description,
+            'is_live': is_live,
+        }
+
+
+class TalkTVSeriesIE(InfoExtractor):
+    IE_NAME = 'talk.tv:series'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?talk\.tv/shows/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
+    _TESTS = [{
+        'url': 'https://watch.talk.tv/shows/86dadc3e-c4d2-11ec-b4c6-0af62ebc70d1',
+        'info_dict': {
+            'id': '86dadc3e-c4d2-11ec-b4c6-0af62ebc70d1',
+        },
+        'playlist_mincount': 4,
+    },
+    ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        episodes = re.finditer(
+            r'''(?i)<a\b[^>]+?\bhref\s*=\s*(?P<q>"|')(?P<href>/watch/(?:(?!(?P=q)).)+)(?P=q)''',
+            webpage)
+        return self.playlist_from_matches(
+            episodes, playlist_id, getter=lambda x: urljoin(url, x.group('href')), ie='TalkTV')

From fed9c8d17b52c722d6e1fc3321f9083ad66ba97d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 28 Apr 2022 11:19:55 +0100
Subject: [PATCH 201/394] Add IE_DESC

---
 youtube_dl/extractor/talktv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/talktv.py b/youtube_dl/extractor/talktv.py
index 75b6acc78cf..6adc584a313 100644
--- a/youtube_dl/extractor/talktv.py
+++ b/youtube_dl/extractor/talktv.py
@@ -26,6 +26,7 @@
 
 class TalkTVIE(InfoExtractor):
     IE_NAME = 'talk.tv'
+    IE_DESC = 'TalkTV UK catch-up and live shows'
     _VALID_URL = r'https?://watch\.talk\.tv/(?P<id>watch/(?:vod|replay)/\d+|live)'
     _TESTS = [{
         'url': 'https://watch.talk.tv/watch/replay/12216792',
@@ -156,6 +157,7 @@ def _real_extract(self, url):
 
 class TalkTVSeriesIE(InfoExtractor):
     IE_NAME = 'talk.tv:series'
+    IE_DESC = 'TalkTV UK series catch-up'
     _VALID_URL = r'https?://(?:watch\.|www\.)?talk\.tv/shows/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
     _TESTS = [{
         'url': 'https://watch.talk.tv/shows/86dadc3e-c4d2-11ec-b4c6-0af62ebc70d1',

From 97d1d589b53eb0a6180ce57b974e637044793c2e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 28 Apr 2022 15:25:49 +0100
Subject: [PATCH 202/394] [doc] Clarify test naming

---
 README.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 2841ed68ff3..cd888c73100 100644
--- a/README.md
+++ b/README.md
@@ -1069,9 +1069,11 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note:
+    * the test names use the extractor class name **without the trailing `IE`**
+    * tests with `only_matching` key in test's dict are not counted.
+8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
 

From 309151cb5e0ee2fb18aafee8bba2cc6a405b21b1 Mon Sep 17 00:00:00 2001
From: Valeriy Litkovskyy <vlr.ltkvsk@protonmail.com>
Date: Sat, 30 Apr 2022 20:10:49 +0200
Subject: [PATCH 203/394] [cozytv] Add new extractor

---
 youtube_dl/extractor/cozytv.py     | 103 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/cozytv.py

diff --git a/youtube_dl/extractor/cozytv.py b/youtube_dl/extractor/cozytv.py
new file mode 100644
index 00000000000..5811e74a51d
--- /dev/null
+++ b/youtube_dl/extractor/cozytv.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    unified_timestamp,
+)
+
+
+class CozyTVReplayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cozy\.tv/(?P<user>[^/]+)/replays/(?P<id>\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])(?:_\d+)?)'
+    _TESTS = [
+        {
+            'url': 'https://cozy.tv/althype/replays/2022-04-22',
+            'md5': '46fb59b29231c816ef4abaa4c54bdbaf',
+            'info_dict': {
+                'id': '6261fed63f2f1a99756b6923',
+                'title': 'Where did the Jews Go?',
+                'ext': 'mp4',
+                'display_id': '2022-04-22',
+                'thumbnail': r're:^https?://.*/replays/althype/2022-04-22/thumb\.webp$',
+                'uploader': 'althype',
+                'creator': 'althype',
+                'release_timestamp': 1650589357,
+                'release_date': '20220422',
+                'timestamp': 1650589357,
+                'upload_date': '20220422',
+                'uploader_id': 'althype',
+                'uploader_url': 'https://cozy.tv/althype',
+                'channel': 'althype',
+                'channel_id': 'althype',
+                'channel_url': 'https://cozy.tv/althype',
+                'duration': 7349,
+                'view_count': 804,
+                'webpage_url': 'https://cozy.tv/althype/replays/2022-04-22',
+            }
+        },
+        {
+            'url': 'https://cozy.tv/beardson/replays/2022-05-01_2',
+            'md5': '2f474ea9404fe90ca87a6390a6e10173',
+            'info_dict': {
+                'id': '626eb2b73f2f1a99756b6a1b',
+                'title': '24 HOUR STREAM',
+                'ext': 'mp4',
+                'display_id': '2022-05-01_2',
+                'thumbnail': r're:^https?://.*/replays/beardson/2022-05-01_2/thumb\.webp$',
+                'uploader': 'beardson',
+                'creator': 'beardson',
+                'release_timestamp': 1651421852,
+                'release_date': '20220501',
+                'timestamp': 1651421852,
+                'upload_date': '20220501',
+                'uploader_id': 'beardson',
+                'uploader_url': 'https://cozy.tv/beardson',
+                'channel': 'beardson',
+                'channel_id': 'beardson',
+                'channel_url': 'https://cozy.tv/beardson',
+                'duration': 313,
+                'view_count': 656,
+                'webpage_url': 'https://cozy.tv/beardson/replays/2022-05-01_2',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id, user = re.match(self._VALID_URL, url).group('id', 'user')
+        data = self._download_json('https://api.cozy.tv/cache/%s/replay/%s' % (user, video_id), video_id)
+
+        cdn_url = '/'.join((data['cdns'][0], 'replays', user, video_id))
+        user_url = 'https://cozy.tv/' + user
+
+        info = {
+            'id': data.get('_id') or video_id,
+            'title': data['title'],
+        }
+
+        timestamp = unified_timestamp(data.get('date'))
+        formats = self._extract_m3u8_formats(cdn_url + '/index.m3u8', video_id, ext='mp4', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        info.update({
+            'channel': user,
+            'channel_id': user,
+            'channel_url': user_url,
+            'creator': user,
+            'display_id': video_id,
+            'duration': float_or_none(data.get('duration')),
+            'formats': formats,
+            'release_timestamp': timestamp,
+            'thumbnail': cdn_url + '/thumb.webp',
+            'timestamp': timestamp,
+            'uploader': user,
+            'uploader_id': user,
+            'uploader_url': user_url,
+            'view_count': int_or_none(data.get('peakViewers')),
+            'webpage_url': url,
+        })
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..7864c71e6a3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -247,6 +247,7 @@
     CNNArticleIE,
 )
 from .coub import CoubIE
+from .cozytv import CozyTVReplayIE
 from .comedycentral import (
     ComedyCentralIE,
     ComedyCentralTVIE,

From 7fa3c37413b366de78ee93f07e212063cfdd7133 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Wed, 4 May 2022 17:53:46 +0200
Subject: [PATCH 204/394] update version

---
 CONTRIBUTING.md        | 8 +++++---
 docs/supportedsites.md | 9 +++++++--
 youtube_dl/version.py  | 2 +-
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 58ab3a4b894..c86c5cb8b28 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -151,9 +151,11 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note:
+    * the test names use the extractor class name **without the trailing `IE`**
+    * tests with `only_matching` key in test's dict are not counted.
+8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f0be2b04dea..9d899ba63b5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -203,6 +203,7 @@
  - **CONtv**
  - **Corus**
  - **Coub**
+ - **CozyTVReplay**
  - **cpac**
  - **cpac:playlist**
  - **Cracked**
@@ -440,6 +441,7 @@
  - **KinoPoisk**
  - **KonserthusetPlay**
  - **KrasView**: Красвью
+ - **KTH**
  - **Ku6**
  - **KUSI**
  - **kuwo:album**: 酷我音乐 - 专辑
@@ -935,6 +937,8 @@
  - **t-online.de**
  - **Tagesschau**
  - **tagesschau:player**
+ - **talk.tv**: TalkTV UK catch-up and live shows
+ - **talk.tv:series**: TalkTV UK series catch-up
  - **Tass**
  - **TBS**
  - **TDSLifeway**
@@ -976,8 +980,9 @@
  - **ThisAmericanLife**
  - **ThisAV**
  - **ThisOldHouse**
- - **TikTok**
- - **TikTokUser** (Currently broken)
+ - **tiktok**: TikTok video extractor
+ - **tiktok:shortcut**: Resolver for TikTok shortcuts
+ - **tiktok:user**: TikTok user profile extractor
  - **tinypic**: tinypic.com videos
  - **TMZ**
  - **TMZArticle**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5624e775964..aefc40d55db 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.15'
+__version__ = '2022.05.04'

From 67c58034efe89efd864fbb9b14fcea8789ced0c3 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Thu, 5 May 2022 07:05:38 +0500
Subject: [PATCH 205/394] fixed issues.

---
 youtube_dl/extractor/wikimedia.py | 87 +++++++++++++++----------------
 1 file changed, 43 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 27c023a6ec4..84c17c2a740 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -1,70 +1,69 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 from .common import InfoExtractor
-from ..utils import get_element_by_class, determine_ext, clean_html, KNOWN_EXTENSIONS
+
 import re
 
+from ..utils import (
+    clean_html,
+    determine_ext,
+    get_element_by_class,
+    urljoin,
+    compat_parse_qs,
+    ExtractorError)
+
 
 class WikimediaIE(InfoExtractor):
-    _NETRC_MACHINE = 'wikimediaorg'
     IE_NAME = 'wikimedia.org'
+    _NETRC_MACHINE = 'wikimediaorg'
     _API_BASE_URL = 'https://commons.wikimedia.org/'
-    _VALID_URL = r'https://commons.wikimedia.org/wiki/File:(?P<id>[^/]+)'
+    _VALID_URL = 'https://commons.wikimedia.org/wiki/File:(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'https://commons.wikimedia.org/wiki/File:Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
         'info_dict': {
-            'description': 'md5:D6F4C7BF1C0DB1EAE80371B1F93EA85E',
-            'ext': 'webm',
-            'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
+            'description': 'md5:7cd84f76e7081f1be033d0b155b4a460',
+            'ext': 'webm', 'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
             'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
-            'license': 'This file is licensed under the Creative Commons Attribution 4.0 International license.',
-            'author': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy',
-            'subtitles': {'de': [
-                {'ext': 'srt',
-                 'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=de&amp&trackformat=srt'}],
-                'en-gb': [
-                    {'ext': 'srt',
-                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=en-gb&amp&trackformat=srt'}],
-                'nl': [
-                    {'ext': 'srt',
-                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp&title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp&lang=nl&amp&trackformat=srt'}
-                ]}
-        }
+            'license': 'md5:62907cddf705a9f7ae7076c15407a977',
+            'author': None, 'subtitles': {'de': [{'ext': 'srt',
+                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=de&amp;trackformat=vtt '}],
+                                          'en-gb': [{'ext': 'srt',
+                                                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=en-gb&amp;trackformat=vtt '}],
+                                          'nl': [{'ext': 'srt',
+                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=nl&amp;trackformat=vtt '}],
+                                          'en': [{'ext': 'srt',
+                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS_-_redub_NL.webm&amp;lang=en&amp;trackformat=vtt '}]}}
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        ext = determine_ext(url)
-        if not ext.lower() in KNOWN_EXTENSIONS:
-            raise Exception("invalid video url")
-
+        ext = determine_ext(url, None)
+        if ext is None:
+            raise ExtractorError('invalid video url', expected=True)
         webpage = self._download_webpage(url, video_id)
         self.report_extraction(video_id)
-        video_url = self._html_search_regex(r'<source [^>]*src="([^"]+)"', webpage,
-                                            u'video URL')
-        licenze = self._html_search_regex(r'\bThis\s*(.*?)\s*license\b', webpage, u'video license')
-        licenze = "This " + licenze + " license."
+        video_url = self._html_search_regex('<source [^>]*src="([^"]+)"', webpage,
+                                            'video URL')
+        license = get_element_by_class('layouttemplate licensetpl mw-content-ltr', webpage)
+        license = clean_html(license)
 
         description = get_element_by_class('description', webpage)
-        author = self._html_search_regex(r'<td>([^\<]*?)<\/td>', str(webpage), u"video author")
-        info = {}
-
-        info['url'] = video_url
-        info['description'] = clean_html(description)
-        info['ext'] = ext
-        info['id'] = video_id.replace('.' + ext, "")
-        info['title'] = self._og_search_title(webpage).replace("File:", "")
-        info['license'] = licenze
-        info['author'] = author
+        author = self._html_search_regex('<td\b[Author>]*>[\n\s]([^<]+?)\s*</td>', webpage, 'video author',
+                                         default=None)
+        info = {'url': video_url, 'description': clean_html(description), 'ext': ext,
+                'id': video_id.replace('.' + ext, ''), 'title': self._og_search_title(webpage).replace('File:', ''),
+                'license': license, 'author': author}
 
         subtitles = {}
-        for sub in re.findall(r'\bsrc=\"\/w\/api\s*(.*?)\s*srt\b', str(webpage)):
-            sub = 'https://commons.wikimedia.org/w/api' + sub + 'srt'
-            lang = sub[sub.find('lang=') + 5:]
-            lang = lang[:lang.find('&')]
-            sub = sub.replace(';', '&')
-            info['subtitles'][lang] = [{"ext": "srt", "url": sub}]
-
+        for sub in re.findall(r'''\bsrc\s*=\s*[\"\'](\/w\/api(.*?)[\s\"])\b''', webpage):
+            sub = sub[0].replace('"', '''''')
+            sub = urljoin('https://commons.wikimedia.org', sub)
+            qs = compat_parse_qs(sub)
+            lang = qs.get('lang', [None])[-1]
+            if not lang:
+                continue
+            subtitles[lang] = [{'ext': 'srt', 'url': sub}]
         info['subtitles'] = subtitles
         return info

From 2e8984e8c221e719eddf587e1cc1ed7bcfcd19d0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 4 May 2022 17:30:07 +0100
Subject: [PATCH 206/394] [common] Extract series name from ld+json types such
 as TVSeries

---
 youtube_dl/extractor/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 797c35fd52d..526d3483ee6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1319,6 +1319,11 @@ def extract_video_object(e):
                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
                         info['series'] = unescapeHTML(part_of_series.get('name'))
+                elif item_type in ('TVSeries', 'Series', 'CreativeWorkSeries'):
+                    series_name = unescapeHTML(e.get('name'))
+                    info.update({
+                        'series': series_name,
+                    })
                 elif item_type == 'Movie':
                     info.update({
                         'title': unescapeHTML(e.get('name')),

From 48c366cd89d321ea9fdc52a10946951634ff042f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 4 May 2022 19:25:41 +0100
Subject: [PATCH 207/394] [common] Avoid erasing extracted ld+json data with
 None values

---
 youtube_dl/extractor/common.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 526d3483ee6..b07b7f9c48f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1228,6 +1228,10 @@ def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
         if isinstance(json_ld, dict):
             json_ld = [json_ld]
 
+        def valued_dict(items):
+            """Return dict from dict or iterable of pairs omitting None values"""
+            return dict((k, v) for k, v in (items.items() if isinstance(items, dict) else items) if v is not None)
+
         INTERACTION_TYPE_MAP = {
             'CommentAction': 'comment',
             'AgreeAction': 'like',
@@ -1325,18 +1329,19 @@ def extract_video_object(e):
                         'series': series_name,
                     })
                 elif item_type == 'Movie':
-                    info.update({
+                    # here and in the next, don't erase existing value with None
+                    info.update(valued_dict({
                         'title': unescapeHTML(e.get('name')),
                         'description': unescapeHTML(e.get('description')),
                         'duration': parse_duration(e.get('duration')),
                         'timestamp': unified_timestamp(e.get('dateCreated')),
-                    })
+                    }))
                 elif item_type in ('Article', 'NewsArticle'):
-                    info.update({
+                    info.update(valued_dict({
                         'timestamp': parse_iso8601(e.get('datePublished')),
                         'title': unescapeHTML(e.get('headline')),
                         'description': unescapeHTML(e.get('articleBody')),
-                    })
+                    }))
                 elif item_type == 'VideoObject':
                     extract_video_object(e)
                     if expected_type is None:
@@ -1350,7 +1355,7 @@ def extract_video_object(e):
                     continue
                 else:
                     break
-        return dict((k, v) for k, v in info.items() if v is not None)
+        return valued_dict(info)
 
     @staticmethod
     def _hidden_inputs(html):

From 66ee6aa2da2d71ababcbce6b7604c29dc83d2c82 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 6 May 2022 03:27:45 +0100
Subject: [PATCH 208/394] [TRT] Add extractor for TRT (Turkish TV/radio)
 catch-up and live streams

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/trt.py        | 325 +++++++++++++++++++++++++++++
 2 files changed, 329 insertions(+)
 create mode 100644 youtube_dl/extractor/trt.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..7d6a0505383 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1293,6 +1293,10 @@
     TrovoIE,
     TrovoVodIE,
 )
+from .trt import (
+    TRTIE,
+    TRTLiveIE,
+)
 from .trunews import TruNewsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
diff --git a/youtube_dl/extractor/trt.py b/youtube_dl/extractor/trt.py
new file mode 100644
index 00000000000..65553427d46
--- /dev/null
+++ b/youtube_dl/extractor/trt.py
@@ -0,0 +1,325 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+import time
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urlparse,
+)
+from ..utils import (
+    determine_ext,
+    dict_get,
+    merge_dicts,
+    parse_iso8601,
+    strip_or_none,
+    try_get,
+    url_or_none,
+    urljoin,
+)
+
+
+class TRTIE(InfoExtractor):
+    IE_DESC = 'TRT (Turkish State TV) programs and series'
+    _VALID_URL = r'''(?x)
+                     (?P<list>https?://(?:www\.)trtizle\.com/
+                         (?:diziler|programlar|belgesel|filmler|cocuk|trtarsiv|engelsiz)/
+                             (?P<show>[\w-]+))(?:/(?P<id>[\w-]+))?'''
+    _TESTS = [{
+        'url': 'https://www.trtizle.com/belgesel/dunya-tarihinin-donum-noktalari/dunya-tarihinin-donum-noktalari-1-bolum-125583',
+        'md5': 'c46dc0b9b53ad372c4ac6b3982805f05',
+        'info_dict': {
+            'id': 'dunya-tarihinin-donum-noktalari-1-bolum-125583',
+            'ext': 'mp4',
+            'title': 'Dünya Tarihinin Dönüm Noktaları 1.Bölüm',
+            'description': 'Bedelini insanların ödeyeceği bir imparatorluk çekişmesinde Persler, Yunanlara karşı...',
+            'timestamp': 1617148800,
+            'upload_date': '20210331',
+            'thumbnail': r're:https?://.+\.jpe?g',
+            'duration': float,
+            'series': 'Dünya Tarihinin Dönüm Noktaları',
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.trtizle.com/belgesel/dunya-tarihinin-donum-noktalari',
+        'info_dict': {
+            'id': 'dunya-tarihinin-donum-noktalari',
+            'title': 'Dünya Tarihinin Dönüm Noktaları',
+        },
+        'playlist_mincount': 22,
+    }, {
+        'url': 'https://www.trtizle.com/diziler/yol-ayrimi/yol-ayrimi-1-bolum-5774583',
+        'md5': '67ada6b2020b5dd0d3e24646b2725676',
+        'info_dict': {
+            'id': 'yol-ayrimi-1-bolum-5774583',
+            'ext': 'mp4',
+            'title': 'Yol Ayrımı 1.Bölüm',
+            'description': 'Seyrisefain balosunda, herkes bir haberin akıbetini beklemektedir…',
+            'timestamp': 1623888000,
+            'upload_date': '20210617',
+            'thumbnail': r're:https?://.+\.jpe?g',
+            'duration': float,
+            'series': 'Yol Ayrımı',
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.trtizle.com/diziler/yol-ayrimi/',
+        'info_dict': {
+            'id': 'yol-ayrimi',
+            'title': 'Yol Ayrımı',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://www.trtizle.com/programlar/sade-saz/sade-saz-1-bolum-7646201',
+        'md5': '8f416e64379ea4d1d3ea0a65dc922f5c',
+        'info_dict': {
+            'id': 'sade-saz-1-bolum-7646201',
+            'ext': 'mp4',
+            'title': 'Sade Saz 1.Bölüm',
+            'description': 'Sade Saz’ın ilk bölümünün konuğu, tanbur icracısı K. Alper Uzkur.',
+            'timestamp': 1641772800,
+            'upload_date': '20220110',
+            'thumbnail': r're:https?://.+\.jpe?g',
+            'duration': float,
+            'series': 'Sade Saz',
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.trtizle.com/programlar/sade-saz',
+        'info_dict': {
+            'id': 'sade-saz',
+            'title': 'Sade Saz',
+        },
+        'playlist_mincount': 6,
+    }, {
+        'url': 'https://www.trtizle.com/filmler/looking-for-eric/looking-for-eric-8414201',
+        'md5': '833d61e4a10606d71b3903295cfa3c63',
+        'info_dict': {
+            'id': 'looking-for-eric-8414201',
+            'ext': 'mp4',
+            'title': 'Looking for Eric',
+            'description': 'Postacı Eric\'in hayatı krize sürüklenirken gerçek ve hayal birbirine karışır...',
+            'upload_date': '20220401',
+            'timestamp': 1648771200,
+            'thumbnail': r're:https?://.+\.jpe?g',
+            'duration': float,
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.trtizle.com/cocuk/kaptan-pengu-ve-arkadaslari/kaptan-pengu-ve-arkadaslari-okul-aciliyor-6034815',
+        'md5': '551c479d1a6bc7c538356907d4ea5d19',
+        'info_dict': {
+            'id': 'kaptan-pengu-ve-arkadaslari-okul-aciliyor-6034815',
+            'ext': 'mp4',
+            'title': 'Kaptan Pengu ve Arkadaşları 1.Bölüm',
+            'description': 'Hayvanlar Konseyi\'nden Kaptan Pengu\'ya bir mektup vardır...',
+            'timestamp': 1626134400,
+            'upload_date': '20210713',
+            'thumbnail': r're:https?://.+\.jpe?g',
+            'duration': float,
+            'series': 'Kaptan Pengu ve Arkadaşları',
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.trtizle.com/cocuk/kaptan-pengu-ve-arkadaslari',
+        'info_dict': {
+            'id': 'kaptan-pengu-ve-arkadaslari',
+            'title': 'Kaptan Pengu ve Arkadaşları',
+        },
+        'playlist_mincount': 41,
+    },
+    ]
+
+    def _extract_formats(self, fmt_url, video_id):
+        formats = []
+        ext = determine_ext(fmt_url)
+        if ext == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                fmt_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        elif ext == 'mpd':
+            formats.extend(self._extract_mpd_formats(
+                fmt_url, video_id, mpd_id='dash', fatal=False))
+        else:
+            formats.append({
+                'url': fmt_url,
+            })
+        return formats
+
+    def _extract_list(self, playlist_id, url):
+        webpage = self._download_webpage(url, playlist_id)
+        LIST_RE = (
+            r'''<a\s[^>]*?\b%s\s*=\s*['"](%s(?:(?<=/)|/)[\w-]+)'''
+            % ('data-path' if 'data-path' in webpage else 'href',
+               re.escape(compat_urlparse.urlparse(url).path), ))
+
+        def entries():
+            for item_url in re.finditer(LIST_RE, webpage):
+                item_url = urljoin(url, item_url.group(1))
+                yield self._extract_video(self._match_id(item_url), item_url)
+
+        series = self._search_json_ld(webpage, playlist_id, default={}, expected_type='TVSeries')
+        return self.playlist_result(entries(), playlist_id, series.get('series'))
+
+    def _extract_video(self, video_id, url):
+        webpage = self._download_webpage(url, video_id)
+        result = self._search_json_ld(webpage, video_id, default={})
+        result['id'] = video_id
+        if 'title' not in result:
+            result['title'] = (
+                self._html_search_meta(('title', 'og:title', 'twitter:title'), webpage)
+                or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title\b', webpage, 'title'))
+        fmt_url = result.get('url')
+        formats = []
+        if fmt_url:
+            del result['url']
+            formats = self._extract_formats(fmt_url, video_id)
+        self._sort_formats(formats)
+        result['formats'] = formats
+
+        return merge_dicts(
+            result, {
+                'description': self._html_search_meta(('description', 'og:description'), webpage, 'description'),
+                'thumbnail': url_or_none(self._og_search_thumbnail(webpage)),
+            })
+
+    def _real_extract(self, url):
+        show_id, video_id, playlist_url = re.match(self._VALID_URL, url).group('show', 'id', 'list')
+        # TODO: adapt --yes/no-playlist to make this work properly
+        # if not video_id or self._downloader.params.get('noplaylist') is False:
+        if not video_id:
+            return self._extract_list(show_id, playlist_url)
+
+        return self._extract_video(video_id, url)
+
+
+class TRTLiveIE(TRTIE):
+    IE_DESC = 'TRT (Turkish State TV and radio) live channels'
+    _VALID_URL = r'https?://(?:www\.)?trtizle\.com/canli/(?:tv/trt-|radyo/(?:radyo-)?)(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.trtizle.com/canli/tv/trt-world',
+        'info_dict': {
+            'id': 'trtworld',
+            'ext': 'mp4',
+            'title': r're:TRT WORLD .+',
+            'description': 'TRT World',
+            'is_live': True,
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        }
+    },
+    ]
+
+    def _real_extract(self, url):
+        chan_id = self._match_id(url)
+        webpage = self._download_webpage(url, chan_id)
+        chan_id = self._search_regex(
+            r'\blivePlayer\s*\.\s*openPlayer\s*\([^)]*?\btrt\.com\.tr/trtportal/(?:[^/]+/)+thumbnails/([\w-]+)\.(?:jp|png)',
+            webpage, 'slug', fatal=False) or chan_id
+        chan_url = self._search_regex(
+            r'''\blivePlayerConfig\s*\.\s*baseEpgUrl\s*=\s*(?P<q>'|")(?P<url>https?://(?:(?!(?P=q)).)+)(?P=q)''',
+            webpage, 'player config', group='url')
+        chan_url = '%s%s.json' % (chan_url, chan_id)
+
+        def maybe_xml2json(src):
+            """Turn unexpected XML returned from an API URL into JSON"""
+            m = re.match(r'''^\s*<\?xml\b(?:[^/>]*?\bencoding\s*=\s*['"](?P<enc>[\w-]+))?[^/>]*\?>\s*(?P<xml><.+>)$''', src)
+            if m:
+
+                # Thanks https://stackoverflow.com/a/63556250 for inspiration
+                ATTR_RE = (
+                    r"""(?s)(?P<avr>\S ?)(?:\s*=\s*(?P<q>['"])(?P<avl>.*?)(?<!\\)(?P=q))?"""
+                )
+
+                def elt_value(attr_str, val_dict):
+                    v = {}
+                    attrs = dict((j.group("avr"), j.groupdict(True).get("avl"))
+                                 for j in re.finditer(ATTR_RE, attr_str.strip()))
+                    if attrs:
+                        v['@attributes'] = attrs
+                    v['@values'] = val_dict
+                    return v
+
+                def xml2dict(xml_str):
+                    elts = re.findall(
+                        r"(?s)<(?P<var>\S )(?P<attr>[^/>]*)(?:(?:>(?P<val>.*?)</(?P=var)>)|(?:/>))",
+                        xml_str,
+                    )
+
+                    if elts:
+                        elts = [{i[0]: elt_value(i[1], xml2dict(i[2]))} for i in elts]
+                        if len(elts) == 1:
+                            return elts[0]
+                        return elts
+                    return xml_str
+
+                try:
+                    return json.dumps(xml2dict(m.group('xml').encode(m.group('enc') or 'utf-8')))
+                except Exception:
+                    pass
+            return src
+
+        chan_info = self._download_json(
+            chan_url, chan_id, fatal=False,
+            note='Downloading player EPG JSON',
+            query={'_': int(time.time() * 1000)},
+            expected_status=403,
+            # errors are returned as XML
+            transform_source=maybe_xml2json)
+        if not isinstance(chan_info, dict) or 'Error' in chan_info:
+            chan_info = self._download_json(
+                'https://trtizle-api.cdn.wp.trt.com.tr/trttv/v3/livestream',
+                chan_id, fatal=False,
+                note='Downloading livestream API JSON',
+                query={'path': compat_urlparse.urlparse(url).path}) or {}
+
+        title = chan_info['channel']['title']
+
+        current = try_get(chan_info, lambda x: x['current'], dict) or {}
+        if current.get('geo_block'):
+            self._downloader.report_warning(
+                '[%s] %s' % (self.IE_NAME, 'Stream is geo-blocked'))
+
+        chan_info = chan_info['channel']
+        fmt_url = dict_get(chan_info, ('url', 'noneDvrUrl'))
+        formats = []
+        if fmt_url:
+            formats = self._extract_formats(fmt_url, chan_id)
+        self._sort_formats(formats)
+
+        start_end = [parse_iso8601(current.get(x)) for x in ('starttime', 'endtime')]
+        if None in start_end:
+            start_end = None
+
+        return {
+            'id': chan_id,
+            'title': self._live_title(current.get('title') or title),
+            'is_live': True,
+            'formats': formats,
+            'description': strip_or_none(chan_info.get('description')),
+            'thumbnail': next((url_or_none(chan_info.get(x))
+                              for x in ('thumbnail', 'thumbnailYoutubeUrl', 'square_logo', 'livestreamLogoUrl')),
+                              None),
+            'timestamp': start_end and start_end[0],
+            'duration': start_end and (start_end[1] - time.time()),
+        }

From 13b0b2d37fb474c2584f3780b143b4c5f2cccf46 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 7 May 2022 16:49:45 -0400
Subject: [PATCH 209/394] [VidLii] Add HD support

Also fix a bug with the view count
---
 youtube_dl/extractor/vidlii.py | 56 +++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index f4774256bd5..9b86efc3e7b 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -4,6 +4,7 @@
 import re
 
 from .common import InfoExtractor
+
 from ..utils import (
     float_or_none,
     get_element_by_id,
@@ -35,6 +36,25 @@ class VidLiiIE(InfoExtractor):
             'categories': ['News & Politics'],
             'tags': ['Vidlii', 'Jan', 'Videogames'],
         }
+    }, {  # HD
+        'url': 'https://www.vidlii.com/watch?v=2Ng8Abj2Fkl',
+        'md5': '450e7da379c884788c3a4fa02a3ce1a4',
+        'info_dict': {
+            'id': '2Ng8Abj2Fkl',
+            'ext': 'mp4',
+            'title': 'test',
+            'description': 'md5:cc55a86032a7b6b3cbfd0f6b155b52e9',
+            'thumbnail': 'https://www.vidlii.com/usfi/thmp/2Ng8Abj2Fkl.jpg',
+            'uploader': 'VidLii',
+            'uploader_url': 'https://www.vidlii.com/user/VidLii',
+            'upload_date': '20200927',
+            'duration': 5,
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Film & Animation'],
+            'tags': ['1', '2'],
+        },
     }, {
         'url': 'https://www.vidlii.com/embed?v=tJluaH4BJ3v&a=0',
         'only_matching': True,
@@ -46,9 +66,31 @@ def _real_extract(self, url):
         webpage = self._download_webpage(
             'https://www.vidlii.com/watch?v=%s' % video_id, video_id)
 
-        video_url = self._search_regex(
-            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1', webpage,
-            'video url', group='url')
+        formats = []
+
+        def add_format(format_url, height=None):
+            height = int(self._search_regex(r"(\d+).mp4",
+                         format_url, "height", default=360))
+
+            formats.append({
+                'url': format_url,
+                'format_id': '%dp' % height if height else None,
+                'height': height,
+            })
+
+        sources = re.findall(
+            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1',
+            webpage)
+
+        try:
+            self._request_webpage(sources[1][1], video_id, 'Checking HD URL')
+            add_format(sources[1][1])
+        except Exception:
+            pass
+
+        add_format(sources[0][1])
+
+        self._sort_formats(formats)
 
         title = self._search_regex(
             (r'<h1>([^<]+)</h1>', r'<title>([^<]+) - VidLii<'), webpage,
@@ -83,9 +125,9 @@ def _real_extract(self, url):
             r'duration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._search_regex(
-            (r'<strong>(\d+)</strong> views',
-             r'Views\s*:\s*<strong>(\d+)</strong>'),
-            webpage, 'view count', fatal=False))
+            (r'<strong>([\d,]+)</strong> views',
+             r'Views\s*:\s*<strong>([\d,]+)</strong>'),
+            webpage, 'view count', fatal=False).replace(",", ""))
 
         comment_count = int_or_none(self._search_regex(
             (r'<span[^>]+id=["\']cmt_num[^>]+>(\d+)',
@@ -109,7 +151,7 @@ def _real_extract(self, url):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From a0832f102f900f106a98b840bb038cf918bce67a Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 7 May 2022 16:57:34 -0400
Subject: [PATCH 210/394] Use compat for HTTPError

---
 youtube_dl/extractor/vidlii.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 9b86efc3e7b..0d3b0961e18 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -14,6 +14,10 @@
     urljoin,
 )
 
+from ..compat import (
+    compat_HTTPError
+)
+
 
 class VidLiiIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vidlii\.com/(?:watch|embed)\?.*?\bv=(?P<id>[0-9A-Za-z_-]{11})'
@@ -85,7 +89,7 @@ def add_format(format_url, height=None):
         try:
             self._request_webpage(sources[1][1], video_id, 'Checking HD URL')
             add_format(sources[1][1])
-        except Exception:
+        except compat_HTTPError:
             pass
 
         add_format(sources[0][1])

From 3c87b05bb06fd1ffe0c200d85d83555b66da5a69 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 7 May 2022 17:00:52 -0400
Subject: [PATCH 211/394] Python 2

---
 youtube_dl/extractor/vidlii.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 0d3b0961e18..4b09270df34 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -14,9 +14,10 @@
     urljoin,
 )
 
-from ..compat import (
-    compat_HTTPError
-)
+try:
+    from urllib.error import HTTPError
+except ImportError:  # Python 2 support
+    from urllib2 import HTTPError
 
 
 class VidLiiIE(InfoExtractor):
@@ -89,7 +90,7 @@ def add_format(format_url, height=None):
         try:
             self._request_webpage(sources[1][1], video_id, 'Checking HD URL')
             add_format(sources[1][1])
-        except compat_HTTPError:
+        except HTTPError:
             pass
 
         add_format(sources[0][1])

From c1844208a2b9d952c877ea393052165a286d9a89 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 7 May 2022 17:03:12 -0400
Subject: [PATCH 212/394] hm?

---
 youtube_dl/extractor/vidlii.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 4b09270df34..86cfe5033f6 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -6,6 +6,7 @@
 from .common import InfoExtractor
 
 from ..utils import (
+    ExtractorError,
     float_or_none,
     get_element_by_id,
     int_or_none,
@@ -14,11 +15,6 @@
     urljoin,
 )
 
-try:
-    from urllib.error import HTTPError
-except ImportError:  # Python 2 support
-    from urllib2 import HTTPError
-
 
 class VidLiiIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vidlii\.com/(?:watch|embed)\?.*?\bv=(?P<id>[0-9A-Za-z_-]{11})'
@@ -90,7 +86,7 @@ def add_format(format_url, height=None):
         try:
             self._request_webpage(sources[1][1], video_id, 'Checking HD URL')
             add_format(sources[1][1])
-        except HTTPError:
+        except ExtractorError:
             pass
 
         add_format(sources[0][1])

From a764865324e990d05e4bbf1ec24bb1f11844ac73 Mon Sep 17 00:00:00 2001
From: cens0r <74016750+cens0r@users.noreply.github.com>
Date: Sun, 8 May 2022 16:38:40 +0200
Subject: [PATCH 213/394] fixes and suggestions

---
 youtube_dl/extractor/camwhoresbay.py | 63 +++++++++++-----------------
 1 file changed, 25 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py
index 2c00e4b2e08..5a432af4105 100644
--- a/youtube_dl/extractor/camwhoresbay.py
+++ b/youtube_dl/extractor/camwhoresbay.py
@@ -5,7 +5,11 @@
 import re
 import itertools
 from ..utils import (
+    int_or_none,
     js_to_json,
+    parse_resolution,
+    sanitize_url,
+    update_url_query,
 )
 
 
@@ -21,31 +25,7 @@ class CamWhoresBayVideoIE(InfoExtractor):
             'title': 'Kirsten_Xxx 2022-01-19',
             'uploader': '789sani',
             'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
-            # 'formats': [{
-            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472\.mp4/\?rnd=.+',
-            #     'height': 360
-            # }, {
-            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_480p\.mp4/\?rnd=.+',
-            #     'height': 480
-            # }, {
-            #     'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
-            #     'height': 720
-            # }]
-            'thumbnails': [{
-                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg',
-                'height': 360
-            }, {
-                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_480p.mp4.jpg',
-                'height': 480
-            }, {
-                'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg',
-                'height': 720
-            }],
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
+            'thumbnail': r're:^https?://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg',
         }
     }
 
@@ -54,11 +34,11 @@ def _parse_formats(flashvars):
         formats = []
         rnd = flashvars.get('rnd')
         for k, v in flashvars.items():
-            if re.match(r"^video_(alt_)?url\d?$", k):
+            if re.match(r'^video_(alt_)?url\d?$', k):
                 f = {
-                    'url': '{}?rnd={}'.format(v, rnd),
-                    'height': int(flashvars['{}_text'.format(k)][:-1])
+                    'url': update_url_query(v, {'rnd': rnd, })
                 }
+                f.update(parse_resolution(flashvars.get(k + '_text')))
                 formats.append(f)
         return formats
 
@@ -66,10 +46,10 @@ def _parse_formats(flashvars):
     def _parse_thumbnails(flashvars):
         thumbnails = []
         for k, v in flashvars.items():
-            if re.match(r"^preview_url\d$", k):
+            if re.match(r'^preview_url\d$', k):
                 t = {
-                    'url': 'https:{}'.format(v),
-                    'height': int(flashvars[k.replace('_url', '_height')])
+                    'url': sanitize_url(v),
+                    'height': int_or_none(flashvars.get(k.replace('_url', '_height'))),
                 }
                 thumbnails.append(t)
         return thumbnails
@@ -77,12 +57,17 @@ def _parse_thumbnails(flashvars):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
+        if re.search(r'(?s)<div class=\"no-player\"', webpage):
+            print("Video is private, skipping.")
+            return
+
         title = self._html_search_regex(r'<div class=\"headline\">\n\s*<h1>(.+?)</h1>', webpage, 'title')
 
         flashvars = self._parse_json(
-            self._search_regex(r'var\s+flashvars\s+=\s+({\n.+});', webpage, 'flashvars', default='{}'),
+            self._search_regex(r'(?s)var\s+flashvars\s*=\s*({.+?})\s*;', webpage, 'flashvars', default='{}'),
             video_id, transform_source=js_to_json)
-        uploader = self._search_regex(r'class=\"avatar\" href=\"http.+/\" title=\"(.+)\"', webpage, 'uploader',
+        uploader = self._search_regex(r'''(?s)\bclass\s*=\s*["']avatar["'][^>]+?\btitle\s*=\s*["'](.+)["']''', webpage, 'uploader',
                                       fatal=False)
 
         formats = self._parse_formats(flashvars)
@@ -101,9 +86,9 @@ def _real_extract(self, url):
 class CamWhoresBayModelIE(CamWhoresBayVideoIE):
     IE_NAME = 'camwhoresbay:model'
     _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/models/(?P<id>[0-9a-z\-]+)/?'
-    _MORE_PAGES_INDICATOR = r'<li\s+class="next"><a\s+href="#videos"'
+    _MORE_PAGES_INDICATOR = r'''<li\s[^>]*\bclass\s*=\s*["']next["'][^>]*><a\s[^>]*href\s*=\s*["']#videos["']'''
     _TITLE = None
-    _TITLE_RE = r'<div\s+class=\"headline\">\n\s*<h1>\s*(.*?)\'s New Videos.+</h1>'
+    _TITLE_RE = r'''(?is)<div\s[^>]+?\bclass\s*=\s*["']headline["'][^>]*>\s*<h1\s[^>]*>\s*(.*?)'s\s+New\s+Videos\b'''
     _BASE_URL_TEMPL = 'https://www.camwhoresbay.com/models/%s/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from='
     _TEST = {
         'url': 'https://www.camwhoresbay.com/models/kirsten-xxx/',
@@ -115,7 +100,7 @@ class CamWhoresBayModelIE(CamWhoresBayVideoIE):
     _PAGE_SIZE = 35
 
     def _extract_list_title(self, webpage):
-        return self._TITLE or self._html_search_regex(
+        return self._html_search_regex(
             self._TITLE_RE, webpage, 'list title', fatal=False)
 
     def _title_and_entries(self, model_id, base_url):
@@ -125,7 +110,9 @@ def _title_and_entries(self, model_id, base_url):
             if page == 1:
                 yield self._extract_list_title(webpage)
 
-            for video_id in re.findall(r'<div\s+class=\"video-item\s*\">\n\s*<a href=\"(.*?)\"\s+title', webpage):
+            for video_id in re.findall(
+                    r'(?is)<div\s[^>]*\bclass\s*=\s*["\'.*?\bvideo\-item\b.*?["\'][^>]*>\s*<a\s[^>]*\bhref\s*=\s*["\'](.*?)["\']',
+                    webpage):
                 yield self.url_result(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
@@ -138,4 +125,4 @@ def _extract_videos(self, list_id, base_url):
 
     def _real_extract(self, url):
         model_id = self._match_id(url)
-        return self._extract_videos(model_id, self._BASE_URL_TEMPL % model_id)
+        return self._extract_videos(model_id, self._BASE_URL_TEMPL % (model_id,))

From a7c1a558edb216b49789e189e0e3d5cc8ca46f51 Mon Sep 17 00:00:00 2001
From: cens0r <74016750+cens0r@users.noreply.github.com>
Date: Sun, 8 May 2022 16:41:44 +0200
Subject: [PATCH 214/394] Update youtube_dl/extractor/camwhoresbay.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/camwhoresbay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py
index 5a432af4105..6d1596644fa 100644
--- a/youtube_dl/extractor/camwhoresbay.py
+++ b/youtube_dl/extractor/camwhoresbay.py
@@ -24,7 +24,7 @@ class CamWhoresBayVideoIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Kirsten_Xxx 2022-01-19',
             'uploader': '789sani',
-            'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
+            'url': r're:^https?://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+',
             'thumbnail': r're:^https?://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg',
         }
     }

From f935e0585bb190015dd8eec05e5ee1d6a0580fca Mon Sep 17 00:00:00 2001
From: Martin Vician <Martin.Vician@ibm.com>
Date: Sun, 8 May 2022 19:11:20 +0100
Subject: [PATCH 215/394] Added thumbnail and timestamp

---
 youtube_dl/extractor/mujrozhlas.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mujrozhlas.py b/youtube_dl/extractor/mujrozhlas.py
index a28559d4a72..53cc5a63137 100644
--- a/youtube_dl/extractor/mujrozhlas.py
+++ b/youtube_dl/extractor/mujrozhlas.py
@@ -5,6 +5,7 @@
 from ..utils import (
     int_or_none,
     clean_html,
+    parse_iso8601
 )
 
 
@@ -16,15 +17,19 @@ class MujRozhlasIE(InfoExtractor):
             'id': 'meteor/meteor-o-nejvetsim-matematikovi-nekonecnem-vesmiru-skakajicim-pavoukovi-hrani-surikat',
             'ext': 'mp3',
             'title': 'Meteor o největším matematikovi, nekonečném vesmíru, skákajícím pavoukovi a hraní surikat',
-            'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka'
+            'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka',
+            'timestamp': 1647272701,
+            'upload_date': '20220314',
         }
     }, {
         'url': 'https://www.mujrozhlas.cz/podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
         'info_dict': {
             'id': 'podcast-vinohradska-12/its-humanitarian-disaster-mariupol-we-want-help-says-msfs-alex-wade',
             'ext': 'mp3',
-            'title': 'Meteor o největším matematikovi, nekonečném vesmíru, skákajícím pavoukovi a hraní surikat',
-            'description': 'Poslechněte si:01:00 Vymírající pták roku07:57 Největší experimentátor všech dob14:06 Největší matematik 20. století27:40 Jak si představit nekonečný vesmír?35:12 Pavouk, který skáče bunjee jumping41:34 Jak si hrají surikaty?Hovoří ornitolog Zdeněk Vermouzek, matematik Václav Chvátal nebo astronom Norbert Werner. Rubriku Stalo se tento den připravil ing. František Houdek.\xa0Z knihy\xa0Pozoruhodné objevy ze světa zvířat čte Zuzana Slavíková.\nPetr Sobotka'
+            'title': 'It\'s a humanitarian disaster in Mariupol. We want to help, says MSF‘s Alex Wade',
+            'description': 'It is a humanitarian catastrophe. People in the southeast of Ukraine are dying in the streets and their neighbors have to bury them in the gardens. Those who are still alive, dont have food, water or medicine. I spoke to Alex Wade, an emergency coordinator for Doctors without Borders, who is in Dnipro. He is doing his best to help Ukrainians who are fleeing the war or staying to fight back.\nMatěj Skalický\n\nEdited by: Kateřina Pospíšilová\nSound design: Tomáš Černý\nResearched by:\xa0Alžběta Jurčová\nMusic:\xa0Martin Hůla',
+            'timestamp': 1647442501,
+            'upload_date': '20220316',
         }
     }]
 
@@ -43,6 +48,8 @@ def _real_extract(self, url):
         duration = audio_info.get('duration')
         description = clean_html(attrs.get('description'))
         audio_url = audio_info.get('url')
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = parse_iso8601(self._og_search_property('updated_time', webpage, fatal=False))
 
         return {
             'id': audio_id,
@@ -51,4 +58,6 @@ def _real_extract(self, url):
             'description': description,
             'duration': int_or_none(duration),
             'vcodec': 'none',
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
         }

From bebb1eb104c9545fa461f087cc34a795ea08ce9a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 9 May 2022 18:24:20 +0100
Subject: [PATCH 216/394] Update youtube_dl/extractor/nhk.py

---
 youtube_dl/extractor/nhk.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index c3b44124da7..46a800e7e89 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -129,6 +129,7 @@ class NhkVodIE(NhkBaseIE):
     }, {
         # video, alphabetic character in ID #29670
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
+        'only_matching': True,
         'info_dict': {
             'id': 'qfjay6cg',
             'ext': 'mp4',

From b811a49719ccc85f6a4390a7d55e8db027e91ba2 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Wed, 11 May 2022 21:25:30 +0200
Subject: [PATCH 217/394] cleaning

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9d899ba63b5..82fdca723ce 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -144,6 +144,8 @@
  - **CamdemyFolder**
  - **CamModels**
  - **CamTube**
+ - **camwhoresbay:model**
+ - **camwhoresbay:video**
  - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
@@ -559,6 +561,7 @@
  - **mtvservices:embedded**
  - **MTVUutisetArticle**
  - **MuenchenTV**: münchen.tv
+ - **MujRozhlas**
  - **mva**: Microsoft Virtual Academy videos
  - **mva:course**: Microsoft Virtual Academy courses
  - **Mwave**
@@ -997,6 +1000,8 @@
  - **Trilulilu**
  - **Trovo**
  - **TrovoVod**
+ - **TRT**: TRT (Turkish State TV) programs and series
+ - **TRTLive**: TRT (Turkish State TV and radio) live channels
  - **TruNews**
  - **TruTV**
  - **Tube8**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index aefc40d55db..1a0d5dc0579 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.05.04'
+__version__ = '2022.05.11'

From 7322dc1a8096a58095bdfeb630bef764eb29543d Mon Sep 17 00:00:00 2001
From: Jordan Weatherby <47519158+JordanWeatherby@users.noreply.github.com>
Date: Thu, 12 May 2022 21:13:26 +0100
Subject: [PATCH 218/394] Added piped instances

---
 youtube_dl/extractor/youtube.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ff6c7b0f8b1..456d72f5bce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -457,6 +457,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
+        # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
+        r'(?:www\.)?piped\.kavin\.rocks',
+        r'(?:www\.)?piped\.silkky\.cloud',
+        r'(?:www\.)?piped\.tokhmi\.xyz',
+        r'(?:www\.)?piped\.moomoo\.me',
+        r'(?:www\.)?il\.ax',
+        r'(?:www\.)?piped\.syncpundit\.com',
+        r'(?:www\.)?piped\.mha\.fi',
+        r'(?:www\.)?piped\.mint\.lgbt',
+        r'(?:www\.)?piped\.privacy\.com\.de',
     )
     _VALID_URL = r"""(?x)^
                      (

From 3bffd20f3391d5c18e5af141b204f800d893a27c Mon Sep 17 00:00:00 2001
From: Daenges <Daenges2@gmail.com>
Date: Mon, 16 May 2022 21:28:19 +0200
Subject: [PATCH 219/394] Add extractor for MegaCartoons

---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/megacartoons.py | 45 ++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 youtube_dl/extractor/megacartoons.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..072ef38aca5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -690,6 +690,7 @@
     MixcloudUserIE,
     MixcloudPlaylistIE,
 )
+from .megacartoons import MegaCartoonsIE
 from .mlb import (
     MLBIE,
     MLBVideoIE,
diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
new file mode 100644
index 00000000000..999594e1e2e
--- /dev/null
+++ b/youtube_dl/extractor/megacartoons.py
@@ -0,0 +1,45 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class MegaCartoonsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megacartoons\.net/(?P<id>[a-z-]+)/'
+    _TEST = {
+        'url': 'https://www.megacartoons.net/help-wanted/',
+        'md5': '4ba9be574f9a17abe0c074e2f955fded',
+        'info_dict': {
+            'id': 'help-wanted',
+            'title': 'help-wanted',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'Help Wanted: Encouraged by his best friend, Patrick Starfish, SpongeBob overcomes his fears and finally applies for that dream job as a fry cook at the Krusty Krab. Challenged by the owner, Mr. Krabs, and his assistant Squidward, to prove himself worthy of the job, SpongeBob rises to the occasion, with the help of one very special spatula, by feeding a sea of ravenous anchovies.'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # The id is equal to the title
+        title = video_id
+        # Video and thumbnail are
+        url_json = json.loads(self._html_search_regex(r'<div.*data-item="(?P<videourls>{.*})".*>', webpage, 'videourls'))
+
+        video_url = url_json['sources'][0]['src']
+        video_type = url_json['sources'][0]['type']
+        video_thumbnail = url_json['splash']
+
+        video_description = self._html_search_regex(r'<p>(?P<videodescription>.*)</p>', webpage, 'videodescription')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'format': video_type,
+            'url': video_url,
+            'thumbnail': video_thumbnail,
+            'description': video_description,
+        }

From 2b128c729ede8353fe4583651008a56f3b308ac4 Mon Sep 17 00:00:00 2001
From: Daenges <Daenges2@gmail.com>
Date: Mon, 16 May 2022 21:37:35 +0200
Subject: [PATCH 220/394] Add further comments

---
 youtube_dl/extractor/megacartoons.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index 999594e1e2e..c2789f4e920 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -21,18 +21,21 @@ class MegaCartoonsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
+        # ID is equal to the episode name
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         # The id is equal to the title
         title = video_id
-        # Video and thumbnail are
+
+        # Video data is stored in a json -> extract it from the raw html
         url_json = json.loads(self._html_search_regex(r'<div.*data-item="(?P<videourls>{.*})".*>', webpage, 'videourls'))
 
-        video_url = url_json['sources'][0]['src']
-        video_type = url_json['sources'][0]['type']
-        video_thumbnail = url_json['splash']
+        video_url = url_json['sources'][0]['src']       # Get the video url
+        video_type = url_json['sources'][0]['type']     # Get the video type -> 'video/mp4'
+        video_thumbnail = url_json['splash']            # Get the thumbnail
 
+        # Every video has a short summary -> save it as description
         video_description = self._html_search_regex(r'<p>(?P<videodescription>.*)</p>', webpage, 'videodescription')
 
         return {

From 041abb92b3ca37ce477e4d4b4f4e6d33e6263338 Mon Sep 17 00:00:00 2001
From: Daenges <Daenges2@gmail.com>
Date: Mon, 16 May 2022 22:32:47 +0200
Subject: [PATCH 221/394] Apply coding conventions

---
 youtube_dl/extractor/megacartoons.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index c2789f4e920..5218c034efb 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -13,10 +13,15 @@ class MegaCartoonsIE(InfoExtractor):
         'md5': '4ba9be574f9a17abe0c074e2f955fded',
         'info_dict': {
             'id': 'help-wanted',
-            'title': 'help-wanted',
+            'title': 'Help Wanted',
             'ext': 'mp4',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'description': 'Help Wanted: Encouraged by his best friend, Patrick Starfish, SpongeBob overcomes his fears and finally applies for that dream job as a fry cook at the Krusty Krab. Challenged by the owner, Mr. Krabs, and his assistant Squidward, to prove himself worthy of the job, SpongeBob rises to the occasion, with the help of one very special spatula, by feeding a sea of ravenous anchovies.'
+            'description': 'Help Wanted: Encouraged by his best friend, Patrick Starfish, '
+                           'SpongeBob overcomes his fears and finally applies for that '
+                           'dream job as a fry cook at the Krusty Krab. Challenged by the '
+                           'owner, Mr. Krabs, and his assistant Squidward, to prove himself '
+                           'worthy of the job, SpongeBob rises to the occasion, with the help '
+                           'of one very special spatula, by feeding a sea of ravenous anchovies.'
         }
     }
 
@@ -25,18 +30,18 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # The id is equal to the title
-        title = video_id
+        # Try to find a good title or fallback to the ID
+        title = self._og_search_title(webpage) or video_id
 
         # Video data is stored in a json -> extract it from the raw html
-        url_json = json.loads(self._html_search_regex(r'<div.*data-item="(?P<videourls>{.*})".*>', webpage, 'videourls'))
+        url_json = json.loads(self._html_search_regex(r'<div.*data-item=["/\'](?P<videourls>{.*})["/\'].*>', webpage, 'videourls'))
 
-        video_url = url_json['sources'][0]['src']       # Get the video url
-        video_type = url_json['sources'][0]['type']     # Get the video type -> 'video/mp4'
-        video_thumbnail = url_json['splash']            # Get the thumbnail
+        video_url = url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage)    # Get the video url
+        video_type = url_json.get('sources')[0].get('type')                                        # Get the video type -> 'video/mp4'
+        video_thumbnail = url_json.get('splash') or self._og_search_thumbnail(webpage)             # Get the thumbnail
 
         # Every video has a short summary -> save it as description
-        video_description = self._html_search_regex(r'<p>(?P<videodescription>.*)</p>', webpage, 'videodescription')
+        video_description = self._html_search_regex(r'<p>(?P<videodescription>.*)</p>', webpage, 'videodescription', fatal=False) or self._og_search_description(webpage)
 
         return {
             'id': video_id,

From 65274df94b482e60db50e915a0d910d0a9704924 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <36196667+EhtishamSabir@users.noreply.github.com>
Date: Wed, 18 May 2022 18:30:33 +0500
Subject: [PATCH 222/394] Update youtube_dl/extractor/wikimedia.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/wikimedia.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 84c17c2a740..3529b432aa7 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -50,7 +50,8 @@ def _real_extract(self, url):
         license = clean_html(license)
 
         description = get_element_by_class('description', webpage)
-        author = self._html_search_regex('<td\b[Author>]*>[\n\s]([^<]+?)\s*</td>', webpage, 'video author',
+        author = self._html_search_regex('>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+?)\s*</td>',
+                                         webpage, 'video author', default=None)
                                          default=None)
         info = {'url': video_url, 'description': clean_html(description), 'ext': ext,
                 'id': video_id.replace('.' + ext, ''), 'title': self._og_search_title(webpage).replace('File:', ''),

From 4a43765c84d7b2fe3589030120700f8a013af252 Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Wed, 18 May 2022 18:32:00 +0500
Subject: [PATCH 223/394] fixed issues.

---
 youtube_dl/extractor/wikimedia.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 3529b432aa7..7942142c322 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -52,7 +52,6 @@ def _real_extract(self, url):
         description = get_element_by_class('description', webpage)
         author = self._html_search_regex('>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+?)\s*</td>',
                                          webpage, 'video author', default=None)
-                                         default=None)
         info = {'url': video_url, 'description': clean_html(description), 'ext': ext,
                 'id': video_id.replace('.' + ext, ''), 'title': self._og_search_title(webpage).replace('File:', ''),
                 'license': license, 'author': author}

From 5df576b5a14514b413daa65ad7ad097d02fd40dd Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Wed, 18 May 2022 18:42:06 +0500
Subject: [PATCH 224/394] fixed issues.

---
 youtube_dl/extractor/wikimedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 7942142c322..364c77a33d0 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -50,7 +50,7 @@ def _real_extract(self, url):
         license = clean_html(license)
 
         description = get_element_by_class('description', webpage)
-        author = self._html_search_regex('>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+?)\s*</td>',
+        author = self._html_search_regex(r'>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+?)\s*</td>',
                                          webpage, 'video author', default=None)
         info = {'url': video_url, 'description': clean_html(description), 'ext': ext,
                 'id': video_id.replace('.' + ext, ''), 'title': self._og_search_title(webpage).replace('File:', ''),

From 4f0ad098b130538eab88f960845c60a714ee0c26 Mon Sep 17 00:00:00 2001
From: d3fault <d3fault@d3fault.net>
Date: Tue, 17 May 2022 04:30:37 -0700
Subject: [PATCH 225/394] [postproc/ffmpeg] fix opus using default quality of 5
 as bitrate (closes #28389)

Fix #28389. When extracting audio, and the containing format isn't 'opus',
but the requested format is, then youtube-dl will call ffmpeg to do the conversion.
Without specifying the preferred quality, youtube-dl defaults to '5'.
Integers 0-9 are currently sent to ffmpeg as the bitrate for 'opus',
which is clearly an error and this commit fixes that. It uses an arbitrary
(specific to opus) scale to convert 0-9 into reasonable bitrates between
6 kb/s to 210 kb/s.... with 5 (the default) giving 96 kb/s
---
 youtube_dl/postprocessor/ffmpeg.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9f76c9d4ed2..d20e1189137 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -21,6 +21,7 @@
     dfxp2srt,
     ISO639Utils,
     replace_extension,
+    compat_str,
 )
 
 
@@ -307,9 +308,16 @@ def run(self, information):
             extension = self._preferredcodec
             more_opts = []
             if self._preferredquality is not None:
-                # The opus codec doesn't support the -aq option
-                if int(self._preferredquality) < 10 and extension != 'opus':
-                    more_opts += ['-q:a', self._preferredquality]
+                if int(self._preferredquality) < 10:
+                    if extension == 'opus':
+                        # The opus codec doesn't support the -q:a option
+                        # Magic number of 24 used to give quality numbers 0-9 *some* meaning (as suggested by "knarrff" on Github)
+                        # 0 (best quality) gives 210 kb/s
+                        # 5 (default) gives 96 kb/s
+                        # 9 (worst quality) gives 6 kb/s, which is the opus minimum (and 0 kb/s wouldn't make any sense)
+                        more_opts += ['-b:a', compat_str(max(6, int((9-int(self._preferredquality)) * 24))) + 'k']
+                    else:
+                        more_opts += ['-q:a', self._preferredquality]
                 else:
                     more_opts += ['-b:a', self._preferredquality + 'k']
             if self._preferredcodec == 'aac':

From 643433deb7a2c4bf92f969e049a291b1919c3528 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Fri, 20 May 2022 04:07:00 -0400
Subject: [PATCH 226/394] Double quotes -> single quotes, backslash to period

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/vidlii.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 86cfe5033f6..3a47619b09b 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -70,8 +70,8 @@ def _real_extract(self, url):
         formats = []
 
         def add_format(format_url, height=None):
-            height = int(self._search_regex(r"(\d+).mp4",
-                         format_url, "height", default=360))
+            height = int(self._search_regex(r'(\d+)\.mp4',
+                         format_url, 'height', default=360))
 
             formats.append({
                 'url': format_url,

From 2e8e7d6d3a76fc0df71487d07874ed4f5ffb9a03 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Fri, 20 May 2022 04:12:54 -0400
Subject: [PATCH 227/394] Import str_to_int from ..utils

---
 youtube_dl/extractor/vidlii.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 3a47619b09b..14b8215708d 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -13,6 +13,7 @@
     strip_or_none,
     unified_strdate,
     urljoin,
+    str_to_int,
 )
 
 

From ef49e32c36fa4440621ce4598703402386c044c2 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Fri, 20 May 2022 04:21:38 -0400
Subject: [PATCH 228/394] Apply suggestions from code review

* tags: ['1', '2'] -> list
* self._check_formats() used for checking HD url
* self._search_regex -> self._html_search_regex

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/vidlii.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index 14b8215708d..ce35be62dd6 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -55,7 +55,7 @@ class VidLiiIE(InfoExtractor):
             'comment_count': int,
             'average_rating': float,
             'categories': ['Film & Animation'],
-            'tags': ['1', '2'],
+            'tags': list,
         },
     }, {
         'url': 'https://www.vidlii.com/embed?v=tJluaH4BJ3v&a=0',
@@ -84,17 +84,17 @@ def add_format(format_url, height=None):
             r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1',
             webpage)
 
-        try:
-            self._request_webpage(sources[1][1], video_id, 'Checking HD URL')
-            add_format(sources[1][1])
-        except ExtractorError:
-            pass
-
-        add_format(sources[0][1])
+        formats = []
+        if len(sources) > 1:
+            formats = [sources[1][1]]
+            self._check_formats(formats, video_id)
+        if len(sources) > 0:
+            formats.append(sources[0][1])
+        map(add_format, formats)
 
         self._sort_formats(formats)
 
-        title = self._search_regex(
+        title = self._html_search_regex(
             (r'<h1>([^<]+)</h1>', r'<title>([^<]+) - VidLii<'), webpage,
             'title')
 

From f9eef66c324885e104ce93841eac0deafda33089 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Fri, 20 May 2022 04:27:57 -0400
Subject: [PATCH 229/394] Add periods to accepted view count, get str_to_int to
 handle them for us

---
 youtube_dl/extractor/vidlii.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index ce35be62dd6..a4a5236985c 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -126,10 +126,10 @@ def add_format(format_url, height=None):
             default=None) or self._search_regex(
             r'duration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
 
-        view_count = int_or_none(self._search_regex(
-            (r'<strong>([\d,]+)</strong> views',
-             r'Views\s*:\s*<strong>([\d,]+)</strong>'),
-            webpage, 'view count', fatal=False).replace(",", ""))
+        view_count = str_to_int(self._html_search_regex(
+            (r'<strong>([\d,.]+)</strong> views',
+             r'Views\s*:\s*<strong>([\d,.]+)</strong>'),
+            webpage, 'view count', fatal=False))
 
         comment_count = int_or_none(self._search_regex(
             (r'<span[^>]+id=["\']cmt_num[^>]+>(\d+)',

From 73771cd768bdff96745f4d80536c0f4e61a7f5e7 Mon Sep 17 00:00:00 2001
From: Daenges <Daenges2@gmail.com>
Date: Sat, 21 May 2022 16:19:18 +0200
Subject: [PATCH 230/394] Commit suggested changes. - Verify description
 through md5 - Implement robust detection of description - Remove format
 attribute to allow auto detection - Allow conditioning of URLs

---
 youtube_dl/extractor/megacartoons.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index 5218c034efb..80c17c100c5 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+from ..utils import url_or_none
 
 from .common import InfoExtractor
 
@@ -16,12 +17,7 @@ class MegaCartoonsIE(InfoExtractor):
             'title': 'Help Wanted',
             'ext': 'mp4',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'description': 'Help Wanted: Encouraged by his best friend, Patrick Starfish, '
-                           'SpongeBob overcomes his fears and finally applies for that '
-                           'dream job as a fry cook at the Krusty Krab. Challenged by the '
-                           'owner, Mr. Krabs, and his assistant Squidward, to prove himself '
-                           'worthy of the job, SpongeBob rises to the occasion, with the help '
-                           'of one very special spatula, by feeding a sea of ravenous anchovies.'
+            'description': 'md5:2c909daa6c6cb16b2d4d791dd1a31632'
         }
     }
 
@@ -36,17 +32,20 @@ def _real_extract(self, url):
         # Video data is stored in a json -> extract it from the raw html
         url_json = json.loads(self._html_search_regex(r'<div.*data-item=["/\'](?P<videourls>{.*})["/\'].*>', webpage, 'videourls'))
 
-        video_url = url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage)    # Get the video url
-        video_type = url_json.get('sources')[0].get('type')                                        # Get the video type -> 'video/mp4'
-        video_thumbnail = url_json.get('splash') or self._og_search_thumbnail(webpage)             # Get the thumbnail
+        video_url = url_or_none(url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage))   # Get the video url
+        video_thumbnail = url_or_none(url_json.get('splash') or self._og_search_thumbnail(webpage))            # Get the thumbnail
 
-        # Every video has a short summary -> save it as description
-        video_description = self._html_search_regex(r'<p>(?P<videodescription>.*)</p>', webpage, 'videodescription', fatal=False) or self._og_search_description(webpage)
+        # Find the <article> class in the html
+        article = self._search_regex(
+            r'(?s)<article\b[^>]*?\bclass\s*=\s*[^>]*?\bpost\b[^>]*>(.+?)</article\b', webpage, 'post', default='')
+
+        # Extract the actual description from it
+        video_description = (self._html_search_regex(r'(?s)<p>\s*([^<]+)\s*</p>', article, 'videodescription', fatal=False)
+                            or self._og_search_description(webpage))
 
         return {
             'id': video_id,
             'title': title,
-            'format': video_type,
             'url': video_url,
             'thumbnail': video_thumbnail,
             'description': video_description,

From 1231ce7d561f7a269ccc7d4b404b47184b63f826 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 21 May 2022 14:52:36 +0100
Subject: [PATCH 231/394] [Go] Improve error handling and diagnostics for
 missing pages

* fix tests
---
 youtube_dl/extractor/go.py | 49 ++++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 0a40b5b479a..dec6a47d178 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -158,6 +158,7 @@ class GoIE(AdobePassIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'The content you’re looking for is not available',
     }, {
         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
         'only_matching': True,
@@ -179,19 +180,19 @@ class GoIE(AdobePassIE):
         'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
         'only_matching': True,
     }, {
-        'url': 'https://disneynow.com/shows/ducktales-disney-channel/season-02/episode-09-the-outlaw-scrooge-mcduck/vdka9680773',
+        # will expire and need to be replaced by a newer episode
+        'url': 'https://disneynow.com/shows/ducktales-disney-channel/season-03/episode-22-the-last-adventure/vdka22464554',
         'info_dict': {
-            'id': 'VDKA9680773',
+            'id': 'VDKA22464554',
             'ext': 'mp4',
-            'season_number': 2,
-            'title': 'The Outlaw Scrooge McDuck!',
-            'episode': 'Episode 9',
-            'season': 'Season 2',
-            'episode_number': 9,
+            'title': 'The Last Adventure!',
+            'description': 'The Duck Family uncovers earth-shattering secrets in a final standoff with FOWL, with the future of adventure hanging in the balance.',
+            'season_number': 3,
+            'episode': 'Episode 22',
+            'season': 'Season 3',
+            'episode_number': 22,
             'age_limit': 7,
             'series': 'DuckTales',
-            'description': 'md5:3d450c1094a5e7544ac094f1e6da2827',
-            'duration': 1348,
             'thumbnail': r're:https?://.*?\.jpg',
         },
         'params': {
@@ -204,30 +205,37 @@ class GoIE(AdobePassIE):
             'title': 'DuckTales TV Show',
             'id': 'SH553923438',
         },
-        'playlist_mincount': 33,
+        'playlist_mincount': 30,
         'expected_warnings': ['Ignoring subtitle tracks', ],
     }]
 
     def _extract_videos(self, brand, video_id='-1', show_id='-1'):
         display_id = video_id if video_id != '-1' else show_id
-        return self._download_json(
+        data = self._download_json(
             'http://api.contents.watchabc.go.com/vp2/ws/contents/4000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
-            display_id)['video']
+            display_id, fatal=False)
+        return try_get(data, lambda x: x['video'], list)
+
+    def _raise_extractor_error(self, video_id, msg, expected=True):
+        raise ExtractorError('%s said: %s: %s' % (self.IE_NAME, video_id, msg), expected=expected)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         sub_domain = re.sub(r'^(?:www\.)?(.+?)(?:\.go)?$', r'\1', mobj.group('sub_domain') or '')
         video_id, display_id = mobj.group('id', 'display_id')
+        display_id = display_id or video_id
         site_info = self._SITE_INFO.get(sub_domain, {})
         brand = site_info.get('brand')
         if not video_id or not site_info:
-            webpage = self._download_webpage(url, display_id or video_id)
+            webpage = self._download_webpage(url, display_id, expected_status=[404])
+            if re.search(r'(?:>|=\s*")Page not found\s', webpage):
+                self._raise_extractor_error(display_id, 'Page not found')
             data = self._parse_json(
                 self._search_regex(
                     r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
                     'data', default='{}'),
-                display_id or video_id, fatal=False)
-            # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
+                display_id, fatal=False)
+            # https://abc.com/shows/ExtractorErrmodern-family/episode-guide/season-01/101-pilot
             layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
             video_id = None
             if layout:
@@ -273,8 +281,13 @@ def _real_extract(self, url):
                         video['url'], 'Go', video.get('id'), video.get('title')))
                 entries.reverse()
                 return self.playlist_result(entries, show_id, show_title)
-        video_data = self._extract_videos(brand, video_id)[0]
-        video_id = video_data['id']
+        video_data = self._extract_videos(brand, video_id)
+        if not video_data:
+            self._raise_extractor_error(video_id, 'No video found')
+        video_id = try_get(video_data, lambda x: x[0]['id'], compat_str)
+        if not video_id:
+            self._raise_extractor_error(video_id, 'No video data', expected=False)
+        video_data = video_data[0]
         title = video_data['title']
 
         formats = []
@@ -315,7 +328,7 @@ def _real_extract(self, url):
                             self.raise_geo_restricted(
                                 error['message'], countries=['US'])
                     error_message = ', '.join([error['message'] for error in errors])
-                    raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
+                    self._raise_extractor_error(video_id, error_message)
                 asset_url += '?' + entitlement['uplynkData']['sessionKey']
                 formats.extend(self._extract_m3u8_formats(
                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))

From b6b629f686b5191a61414b0c8186312de9359a80 Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 21 May 2022 16:11:15 -0400
Subject: [PATCH 232/394] TypeError: `formats` needs to be a dictionary, not a
 list

---
 youtube_dl/extractor/vidlii.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index a4a5236985c..fc43d07945d 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -86,12 +86,11 @@ def add_format(format_url, height=None):
 
         formats = []
         if len(sources) > 1:
-            formats = [sources[1][1]]
-            self._check_formats(formats, video_id)
+            add_format(sources[1][1])
         if len(sources) > 0:
-            formats.append(sources[0][1])
-        map(add_format, formats)
+            add_format(sources[0][1])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From dbedd602487ae26148477f8cd05475413f4e9a0e Mon Sep 17 00:00:00 2001
From: Paper <37962225+mrpapersonic@users.noreply.github.com>
Date: Sat, 21 May 2022 16:24:34 -0400
Subject: [PATCH 233/394] Move source check back up to the if statement

We don't need to check if the 360p source is there
---
 youtube_dl/extractor/vidlii.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index fc43d07945d..390d7fe5c2a 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -87,10 +87,10 @@ def add_format(format_url, height=None):
         formats = []
         if len(sources) > 1:
             add_format(sources[1][1])
+            self._check_formats(formats, video_id)
         if len(sources) > 0:
             add_format(sources[0][1])
 
-        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From cf4a829c138c100000086cb55c2b772fe2db47ac Mon Sep 17 00:00:00 2001
From: Daenges <Daenges2@gmail.com>
Date: Mon, 23 May 2022 19:21:56 +0200
Subject: [PATCH 234/394] Implement _search_json_ld()

---
 youtube_dl/extractor/megacartoons.py | 85 ++++++++++++++++++++++------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index 80c17c100c5..ea6161b7bab 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -2,7 +2,16 @@
 from __future__ import unicode_literals
 
 import json
-from ..utils import url_or_none
+import re
+
+from ..utils import (
+    bug_reports_message,
+    JSON_LD_RE,
+    merge_dicts,
+    NO_DEFAULT,
+    RegexNotFoundError,
+    url_or_none,
+)
 
 from .common import InfoExtractor
 
@@ -14,39 +23,81 @@ class MegaCartoonsIE(InfoExtractor):
         'md5': '4ba9be574f9a17abe0c074e2f955fded',
         'info_dict': {
             'id': 'help-wanted',
-            'title': 'Help Wanted',
             'ext': 'mp4',
+            'title': 'Help Wanted - SpongeBob SquarePants',
+            'upload_date': '20200223',
+            'timestamp': 1582416000,
             'thumbnail': r're:^https?://.*\.jpg$',
             'description': 'md5:2c909daa6c6cb16b2d4d791dd1a31632'
         }
     }
 
+    # adapted from common.py pending yt-dlp back-port
+    def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
+        json_ld_list = list(re.finditer(JSON_LD_RE, html))
+        default = kwargs.get('default', NO_DEFAULT)
+        fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
+        json_ld = []
+        for mobj in json_ld_list:
+            json_ld_item = self._parse_json(
+                mobj.group('json_ld'), video_id, fatal=fatal)
+            if not json_ld_item:
+                continue
+            if isinstance(json_ld_item, dict):
+                json_ld.append(json_ld_item)
+            elif isinstance(json_ld_item, (list, tuple)):
+                json_ld.extend(json_ld_item)
+        if json_ld:
+            # handle initial '@graph' with one level of children
+            if len(json_ld) > 0 and '@graph' in json_ld[0] and '@context' in json_ld[0]:
+                # should always be hit here
+                context = json_ld[0]['@context']
+                json_ld_g = json_ld[0]['@graph'] or []
+                for item in json_ld_g:
+                    item.setdefault('@context', context)
+                json_ld = json_ld_g + json_ld[1:]
+            json_ld = self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
+        if json_ld:
+            return json_ld
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
+            raise RegexNotFoundError('Unable to extract JSON-LD')
+        else:
+            self.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
+            return {}
+
     def _real_extract(self, url):
         # ID is equal to the episode name
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # Try to find a good title or fallback to the ID
-        title = self._og_search_title(webpage) or video_id
+        info = self._search_json_ld(webpage, video_id, fatal=False) or {}
+
+        info.update({
+            'id': video_id,
+            # Try to find a good title or fallback to the ID
+            'title': info.get('title') or self._og_search_title(webpage) or video_id.replace('-', ' ').capitalize(),
+        })
 
-        # Video data is stored in a json -> extract it from the raw html
-        url_json = json.loads(self._html_search_regex(r'<div.*data-item=["/\'](?P<videourls>{.*})["/\'].*>', webpage, 'videourls'))
+        if 'url' not in info or 'thumbnail' not in info:
+            # Video data is stored in a json -> extract it from the raw html
+            url_json = json.loads(self._html_search_regex(r'<div.*data-item=["/\'](?P<videourls>{.*})["/\'].*>', webpage, 'videourls'))
 
-        video_url = url_or_none(url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage))   # Get the video url
-        video_thumbnail = url_or_none(url_json.get('splash') or self._og_search_thumbnail(webpage))            # Get the thumbnail
+            video_url = url_or_none(url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage))   # Get the video url
+            video_thumbnail = url_or_none(url_json.get('splash') or self._og_search_thumbnail(webpage))            # Get the thumbnail
+            info = merge_dicts(info, {
+                'url': video_url,
+                'thumbnail': video_thumbnail,
+            })
 
         # Find the <article> class in the html
         article = self._search_regex(
             r'(?s)<article\b[^>]*?\bclass\s*=\s*[^>]*?\bpost\b[^>]*>(.+?)</article\b', webpage, 'post', default='')
 
         # Extract the actual description from it
-        video_description = (self._html_search_regex(r'(?s)<p>\s*([^<]+)\s*</p>', article, 'videodescription', fatal=False)
-                            or self._og_search_description(webpage))
+        info['description'] = (
+            self._html_search_regex(r'(?s)<p>\s*([^<]+)\s*</p>', article, 'videodescription', fatal=False)
+            or self._og_search_description(webpage))
 
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'thumbnail': video_thumbnail,
-            'description': video_description,
-        }
+        return info

From 1c2c830d6f409f2aadf30919b97f419ed1aa8e75 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Thu, 26 May 2022 12:23:48 +0200
Subject: [PATCH 235/394] cleaning

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 82fdca723ce..9b2ac3766ef 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -516,6 +516,7 @@
  - **MediasiteCatalog**
  - **MediasiteNamedCatalog**
  - **Medici**
+ - **MegaCartoons**
  - **megaphone.fm**: megaphone.fm embedded players
  - **Meipai**: 美拍
  - **MelonVOD**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1a0d5dc0579..309692e2d0b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.05.11'
+__version__ = '2022.05.26'

From 181112c218115871aee992615d7db77f6ffa1db8 Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Fri, 27 May 2022 23:34:14 +0200
Subject: [PATCH 236/394] Fixed all recent videos downloading with .m3u8
 extension

---
 youtube_dl/extractor/dumpert.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index d9d9afdecba..a73cd85a99a 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -50,6 +50,7 @@ def _real_extract(self, url):
                 'url': uri,
                 'format_id': version,
                 'quality': quality(version),
+                'ext': 'mp4',
             })
         self._sort_formats(formats)
 

From d5e9a95cce6588679ca201b2f98352f2ccac8545 Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Sat, 28 May 2022 00:20:39 +0200
Subject: [PATCH 237/394] Added test case for 'm3u8-video'

---
 youtube_dl/extractor/dumpert.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index a73cd85a99a..c2414e65b1b 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -29,6 +29,16 @@ class DumpertIE(InfoExtractor):
     }, {
         'url': 'http://legacy.dumpert.nl/embed/6675421/dc440fe7',
         'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/item/100031688_b317a185',
+        'md5': '98bece7add22c2645eb3906e6d55318b',
+        'info_dict': {
+            'id': '100031688/b317a185',
+            'ext': 'mp4',
+            'title': 'Epic schijnbeweging',
+            'description': '<p>Die zag je niet eh</p>',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
+        },
     }]
 
     def _real_extract(self, url):

From bb4d3baa6759247f5ae374505cc96dd00b5da44b Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Sat, 28 May 2022 00:21:29 +0200
Subject: [PATCH 238/394] Updated URL regex and tests to include new URLs valid
 since the new site layout

---
 youtube_dl/extractor/dumpert.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index c2414e65b1b..4eebcdc23bc 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -9,7 +9,7 @@
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'(?P<protocol>https?)://(?:(?:www|legacy)\.)?dumpert\.nl/(?:mediabase|embed|item)/(?P<id>[0-9]+[/_][0-9a-zA-Z]+)'
+    _VALID_URL = r'(?P<protocol>https?)://(?:(?:www|legacy)\.)?dumpert\.nl(?:/(?:mediabase|embed|item)/|(?:/toppers|/latest|/?)\?selectedId=)(?P<id>[0-9]+[/_][0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'https://www.dumpert.nl/item/6646981_951bc60f',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -39,6 +39,15 @@ class DumpertIE(InfoExtractor):
             'description': '<p>Die zag je niet eh</p>',
             'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
         },
+    }, {
+        'url': 'https://www.dumpert.nl/toppers?selectedId=100031688_b317a185',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/latest?selectedId=100031688_b317a185',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/?selectedId=100031688_b317a185',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ddc2ecd23b2138316706e7d8968bb252ce33b5c5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 30 May 2022 02:46:14 +0100
Subject: [PATCH 239/394] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index bbdf6949007..ff40cef784f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, on the condition the class ends with IE.
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):

From 71e630eebfcab3962dbf8523af9dc2628ab8119d Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Mon, 30 May 2022 11:39:50 +0200
Subject: [PATCH 240/394] Changed m3u8 format handling

---
 youtube_dl/extractor/dumpert.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 4eebcdc23bc..4faf824c88b 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -3,6 +3,7 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     qualities,
 )
@@ -65,12 +66,17 @@ def _real_extract(self, url):
             if not uri:
                 continue
             version = variant.get('version')
-            formats.append({
-                'url': uri,
-                'format_id': version,
-                'quality': quality(version),
-                'ext': 'mp4',
-            })
+            ext = determine_ext(uri)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    uri, video_id, ext='mp4', m3u8_id=format_id))
+            else:
+                formats.append({
+                    'url': uri,
+                    'format_id': version,
+                })
+            for format in formats:
+                format['quality'] = quality(format['format_id'])
         self._sort_formats(formats)
 
         thumbnails = []

From edfcc50a7c78a03feb8a99e6e38f5396441ea165 Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Mon, 30 May 2022 11:40:10 +0200
Subject: [PATCH 241/394] Added 1080p format to qualities

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 4faf824c88b..ea92602a897 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -59,7 +59,7 @@ def _real_extract(self, url):
         title = item['title']
         media = next(m for m in item['media'] if m.get('mediatype') == 'VIDEO')
 
-        quality = qualities(['flv', 'mobile', 'tablet', '720p'])
+        quality = qualities(['flv', 'mobile', 'tablet', '720p', '1080p'])
         formats = []
         for variant in media.get('variants', []):
             uri = variant.get('uri')

From 7784d3c37ecb20b3aff157878b1a6c758b4529ac Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Mon, 30 May 2022 11:46:41 +0200
Subject: [PATCH 242/394] Fixed wrong variable name that shouldn't have been
 there

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index ea92602a897..6f49b3d1514 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -69,7 +69,7 @@ def _real_extract(self, url):
             ext = determine_ext(uri)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    uri, video_id, ext='mp4', m3u8_id=format_id))
+                    uri, video_id, ext='mp4', m3u8_id=version))
             else:
                 formats.append({
                     'url': uri,

From 60bbab10223b13de11c30f0437aaacc175132870 Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Mon, 30 May 2022 15:10:04 +0200
Subject: [PATCH 243/394] Changed quality parameter to preference to accodomate
 sorting the m3u8 streams

---
 youtube_dl/extractor/dumpert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 6f49b3d1514..7a42273ba06 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -66,17 +66,17 @@ def _real_extract(self, url):
             if not uri:
                 continue
             version = variant.get('version')
+            preference = quality(version)
             ext = determine_ext(uri)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    uri, video_id, ext='mp4', m3u8_id=version))
+                    uri, video_id, ext='mp4', preference=preference))
             else:
                 formats.append({
                     'url': uri,
                     'format_id': version,
+                    'preference': preference,
                 })
-            for format in formats:
-                format['quality'] = quality(format['format_id'])
         self._sort_formats(formats)
 
         thumbnails = []

From a7de6d54ede320999aa69d199ce3c7882edca27d Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Mon, 30 May 2022 15:39:32 +0200
Subject: [PATCH 244/394] Changed to m3u8_native entry protocol and adjusted
 md5 accordingly

---
 youtube_dl/extractor/dumpert.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 7a42273ba06..9b8735d75ac 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -32,7 +32,7 @@ class DumpertIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'https://www.dumpert.nl/item/100031688_b317a185',
-        'md5': '98bece7add22c2645eb3906e6d55318b',
+        'md5': '34a0c10c91d281141b959b27ee862b8f',
         'info_dict': {
             'id': '100031688/b317a185',
             'ext': 'mp4',
@@ -70,7 +70,8 @@ def _real_extract(self, url):
             ext = determine_ext(uri)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    uri, video_id, ext='mp4', preference=preference))
+                    uri, video_id, ext='mp4', entry_protocol='m3u8_native',
+                    preference=preference))
             else:
                 formats.append({
                     'url': uri,

From 8a0905798e1716100ed3c39b397df3a4012a2f90 Mon Sep 17 00:00:00 2001
From: joshwd36 <joshwd36@gmail.com>
Date: Thu, 2 Jun 2022 17:33:02 +0100
Subject: [PATCH 245/394] Add MacProVideo extractor

---
 docs/supportedsites.md              |  1 +
 youtube_dl/extractor/extractors.py  |  4 ++
 youtube_dl/extractor/macprovideo.py | 86 +++++++++++++++++++++++++++++
 3 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/macprovideo.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ae2a6b8b02b..e9d8d416d00 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -482,6 +482,7 @@
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
  - **m6**
+ - **MacProVideo.com**
  - **mailru**: Видео@Mail.Ru
  - **mailru:music**: Музыка@Mail.Ru
  - **mailru:music:search**: Музыка@Mail.Ru
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..ade0d31eafc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -635,6 +635,10 @@
     LyndaCourseIE
 )
 from .m6 import M6IE
+from .macprovideo import (
+    MacProVideoIE,
+    MacProVideoCourseIE
+)
 from .mailru import (
     MailRuIE,
     MailRuMusicIE,
diff --git a/youtube_dl/extractor/macprovideo.py b/youtube_dl/extractor/macprovideo.py
new file mode 100644
index 00000000000..a3c84c09d24
--- /dev/null
+++ b/youtube_dl/extractor/macprovideo.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from youtube_dl.utils import try_get, str_or_none
+
+from .common import InfoExtractor
+
+
+class MacProVideoBaseIE(InfoExtractor):
+    def _get_video_info(self, data):
+        meta = data.get('ned_programmanifest')
+        video_id = str_or_none(meta.get('fk_videoMediaID'))
+
+        formats = self._extract_m3u8_formats(
+            data['video_medium']['mediaURL'], video_id, 'mp4', m3u8_id='hls', fatal=False)
+        self._sort_formats(formats)
+
+        description = try_get(data, lambda x: x['ned_programmanifest_description']['description'])
+        thumbnail = try_get(data, lambda x: x['thumbnail_medium']['mediaURL'])
+
+        return {
+            'id': video_id,
+            'title': meta.get('vid_title'),
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': meta.get('vid_duration'),
+            'formats': formats
+        }
+
+
+class MacProVideoIE(MacProVideoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?macprovideo\.com/video/(?P<course>[0-9a-zA-Z-]+)/(?P<num>[0-9]+)-[0-9a-zA-Z-]+'
+    _TEST = {
+        'url': 'https://www.macprovideo.com/video/apple-logic-pro-102-recording-and-editing-audio/1-1-transducers-and-converters',
+        'md5': 'md5:608d7c8cf9f31202cbd269273df62aaf',
+        'info_dict': {
+            'id': '319676',
+            'ext': 'mp4',
+            'title': '1. Transducers and Converters',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:c16dc8a42252e2be44225f129c591c3d',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+            'format': 'bestvideo'
+        },
+    }
+
+    def _real_extract(self, url):
+        course, num = re.match(self._VALID_URL, url).groups()
+        num = int(num) - 1
+        data = self._download_json('https://www.macprovideo.com/ajax/videos/' + course,
+                                   None)['data']['videos'][num]
+        return self._get_video_info(data)
+
+
+class MacProVideoCourseIE(MacProVideoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?macprovideo\.com/course/(?P<id>[0-9a-zA-Z-]+)'
+    _TEST = {
+        'url': 'https://www.macprovideo.com/course/apple-logic-pro-102-recording-and-editing-audio',
+        'md5': 'md5:608d7c8cf9f31202cbd269273df62aaf',
+        'info_dict': {
+            'id': '319676',
+            'ext': 'mp4',
+            'title': '1. Transducers and Converters',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:c16dc8a42252e2be44225f129c591c3d',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+            'format': 'bestvideo'
+        },
+    }
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+        videos = self._download_json('https://www.macprovideo.com/ajax/videos/' + course_id, None)['data']['videos']
+        title = try_get(videos, lambda x: x[0]['tutorial']['tu_title'])
+
+        return self.playlist_result(
+            [self._get_video_info(data) for data in videos],
+            playlist_id=course_id, playlist_title=title)

From 8651fb67757da79d1f9bd3401b67a1e61506fa7c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 2 Jun 2022 21:04:01 +0100
Subject: [PATCH 246/394] [Soundcloud] Back-port from yt-dlp

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/soundcloud.py | 428 +++++++++++++++++++----------
 2 files changed, 289 insertions(+), 140 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..aeabd373f9f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1137,6 +1137,7 @@
     SoundcloudUserIE,
     SoundcloudTrackStationIE,
     SoundcloudPlaylistIE,
+    SoundcloudRelatedIE,
     SoundcloudSearchIE,
 )
 from .soundgasm import (
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index abb85e1e501..c7c555929df 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -3,6 +3,7 @@
 
 import itertools
 import re
+import json
 
 from .common import (
     InfoExtractor,
@@ -22,12 +23,14 @@
     int_or_none,
     KNOWN_EXTENSIONS,
     mimetype2ext,
+    remove_end,
     str_or_none,
     try_get,
     unified_timestamp,
     update_url_query,
     url_or_none,
     urlhandle_detect_ext,
+    sanitized_Request,
 )
 
 
@@ -55,7 +58,152 @@ def _real_extract(self, url):
         return self.url_result(api_url)
 
 
-class SoundcloudIE(InfoExtractor):
+class SoundcloudBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'soundcloud'
+
+    _API_V2_BASE = 'https://api-v2.soundcloud.com/'
+    _BASE_URL = 'https://soundcloud.com/'
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
+    _API_AUTH_QUERY_TEMPLATE = '?client_id=%s'
+    _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s'
+    _API_VERIFY_AUTH_TOKEN = 'https://api-auth.soundcloud.com/connect/session%s'
+    _access_token = None
+    _HEADERS = {}
+
+    def _store_client_id(self, client_id):
+        self._downloader.cache.store('soundcloud', 'client_id', client_id)
+
+    def _update_client_id(self):
+        webpage = self._download_webpage('https://soundcloud.com/', None)
+        for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
+            script = self._download_webpage(src, None, fatal=False)
+            if script:
+                client_id = self._search_regex(
+                    r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
+                    script, 'client id', default=None)
+                if client_id:
+                    self._CLIENT_ID = client_id
+                    self._store_client_id(client_id)
+                    return
+        raise ExtractorError('Unable to extract client id')
+
+    def _download_json(self, *args, **kwargs):
+        non_fatal = kwargs.get('fatal') is False
+        if non_fatal:
+            del kwargs['fatal']
+        query = kwargs.get('query', {}).copy()
+        for _ in range(2):
+            query['client_id'] = self._CLIENT_ID
+            kwargs['query'] = query
+            try:
+                return super(SoundcloudBaseIE, self)._download_json(*args, **compat_kwargs(kwargs))
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+                    self._store_client_id(None)
+                    self._update_client_id()
+                    continue
+                elif non_fatal:
+                    self.report_warning(error_to_compat_str(e))
+                    return False
+                raise
+
+    def _initialize_pre_login(self):
+        self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'a3e059563d7fd3372b49b37f00a00bcf'
+
+    def _perform_login(self, username, password):
+        if username != 'oauth':
+            self.report_warning(
+                'Login using username and password is not currently supported. '
+                'Use "--username oauth --password <oauth_token>" to login using an oauth token')
+        self._access_token = password
+        query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID
+        payload = {'session': {'access_token': self._access_token}}
+        token_verification = sanitized_Request(self._API_VERIFY_AUTH_TOKEN % query, json.dumps(payload).encode('utf-8'))
+        response = self._download_json(token_verification, None, note='Verifying login token...', fatal=False)
+        if response is not False:
+            self._HEADERS = {'Authorization': 'OAuth ' + self._access_token}
+            self.report_login()
+        else:
+            self.report_warning('Provided authorization token seems to be invalid. Continue as guest')
+
+        r'''
+        def genDevId():
+            def genNumBlock():
+                return ''.join([str(random.randrange(10)) for i in range(6)])
+            return '-'.join([genNumBlock() for i in range(4)])
+
+        payload = {
+            'client_id': self._CLIENT_ID,
+            'recaptcha_pubkey': 'null',
+            'recaptcha_response': 'null',
+            'credentials': {
+                'identifier': username,
+                'password': password
+            },
+            'signature': self.sign(username, password, self._CLIENT_ID),
+            'device_id': genDevId(),
+            'user_agent': self._USER_AGENT
+        }
+
+        query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID
+        login = sanitized_Request(self._API_AUTH_URL_PW % query, json.dumps(payload).encode('utf-8'))
+        response = self._download_json(login, None)
+        self._access_token = response.get('session').get('access_token')
+        if not self._access_token:
+            self.report_warning('Unable to get access token, login may has failed')
+        else:
+            self._HEADERS = {'Authorization': 'OAuth ' + self._access_token}
+        '''
+
+    def _real_initialize(self):
+        self._initialize_pre_login()
+        username, password = self._get_login_info()
+        if username:
+            self._perform_login(username, password)
+
+        super(SoundcloudBaseIE, self)._real_initialize()
+
+    # signature generation
+    def sign(self, user, pw, clid):
+        a = 33
+        i = 1
+        s = 440123
+        w = 117
+        u = 1800000
+        l = 1042
+        b = 37
+        k = 37
+        c = 5
+        n = '0763ed7314c69015fd4a0dc16bbf4b90'  # _KEY
+        y = '8'  # _REV
+        r = self._USER_AGENT
+        e = user  # _USERNAME
+        t = clid  # _CLIENT_ID
+
+        d = '-'.join([compat_str(mInt) for mInt in [a, i, s, w, u, l, b, k]])
+        p = n + y + d + r + e + t + d + n
+        h = p
+
+        m = 8011470
+        f = 0
+
+        for f in range(f, len(h)):
+            m = (m >> 1) + ((1 & m) << 23)
+            m += ord(h[f])
+            m &= 16777215
+
+        # c is not even needed
+        # out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c)
+        out = '%s:%s:%x:%d' % (y, d, m, c)
+
+        return out
+
+    @classmethod
+    def _resolv_url(cls, url):
+        return cls._API_V2_BASE + 'resolve?url=' + url
+
+
+class SoundcloudIE(SoundcloudBaseIE):
     """Information extractor for soundcloud.com
        To access the media, the uid of the song and a stream token
        must be extracted from the page source and the script must make
@@ -69,8 +217,9 @@ class SoundcloudIE(InfoExtractor):
                             (?!stations/track)
                             (?P<uploader>[\w\d-]+)/
                             (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
-                            (?P<title>[\w\d-]+)/?
-                            (?P<token>[^?]+?)?(?:[?].*)?$)
+                            (?P<title>[\w\d-]+)
+                            (?:/(?P<token>(?!(?:albums|sets|recommended))[^?]+?))?
+                            (?:[?].*)?$)
                        |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
                           (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
                     )
@@ -161,23 +310,16 @@ class SoundcloudIE(InfoExtractor):
         },
         # downloadable song
         {
-            'url': 'https://soundcloud.com/oddsamples/bus-brakes',
-            'md5': '7624f2351f8a3b2e7cd51522496e7631',
+            'url': 'https://soundcloud.com/the80m/the-following',
+            'md5': '9ffcddb08c87d74fb5808a3c183a1d04',
             'info_dict': {
-                'id': '128590877',
-                'ext': 'mp3',
-                'title': 'Bus Brakes',
-                'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
-                'uploader': 'oddsamples',
-                'uploader_id': '73680509',
-                'timestamp': 1389232924,
-                'upload_date': '20140109',
-                'duration': 17.346,
-                'license': 'cc-by-sa',
-                'view_count': int,
-                'like_count': int,
-                'comment_count': int,
-                'repost_count': int,
+                'id': '343609555',
+                'ext': 'wav',
+                'title': 'The Following',
+                'timestamp': 1506120436,
+                'upload_date': '20170922',
+                'uploader_id': '312384765',
+                'uploader': '80M',
             },
         },
         # private link, downloadable format
@@ -233,8 +375,8 @@ class SoundcloudIE(InfoExtractor):
                 'id': '583011102',
                 'ext': 'mp3',
                 'title': 'Mezzo Valzer',
-                'description': 'md5:4138d582f81866a530317bae316e8b61',
-                'uploader': 'Micronie',
+                'description': 'md5:8de664f12895716c8ac6f1da6348eb8e',
+                'uploader': 'Giovanni Sarani',
                 'uploader_id': '3352531',
                 'timestamp': 1551394171,
                 'upload_date': '20190228',
@@ -248,14 +390,17 @@ class SoundcloudIE(InfoExtractor):
             },
         },
         {
-            # with AAC HQ format available via OAuth token
+            # with AAC HQ format available via OAuth token (account with active subscription needed)
             'url': 'https://soundcloud.com/wandw/the-chainsmokers-ft-daya-dont-let-me-down-ww-remix-1',
             'only_matching': True,
         },
+        {
+            # Go+ (account with active subscription needed)
+            'url': 'https://soundcloud.com/taylorswiftofficial/look-what-you-made-me-do',
+            'only_matching': True,
+        },
     ]
 
-    _API_V2_BASE = 'https://api-v2.soundcloud.com/'
-    _BASE_URL = 'https://soundcloud.com/'
     _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
 
     _ARTWORK_MAP = {
@@ -271,50 +416,6 @@ class SoundcloudIE(InfoExtractor):
         'original': 0,
     }
 
-    def _store_client_id(self, client_id):
-        self._downloader.cache.store('soundcloud', 'client_id', client_id)
-
-    def _update_client_id(self):
-        webpage = self._download_webpage('https://soundcloud.com/', None)
-        for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
-            script = self._download_webpage(src, None, fatal=False)
-            if script:
-                client_id = self._search_regex(
-                    r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
-                    script, 'client id', default=None)
-                if client_id:
-                    self._CLIENT_ID = client_id
-                    self._store_client_id(client_id)
-                    return
-        raise ExtractorError('Unable to extract client id')
-
-    def _download_json(self, *args, **kwargs):
-        non_fatal = kwargs.get('fatal') is False
-        if non_fatal:
-            del kwargs['fatal']
-        query = kwargs.get('query', {}).copy()
-        for _ in range(2):
-            query['client_id'] = self._CLIENT_ID
-            kwargs['query'] = query
-            try:
-                return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                    self._store_client_id(None)
-                    self._update_client_id()
-                    continue
-                elif non_fatal:
-                    self._downloader.report_warning(error_to_compat_str(e))
-                    return False
-                raise
-
-    def _real_initialize(self):
-        self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk'
-
-    @classmethod
-    def _resolv_url(cls, url):
-        return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
-
     def _extract_info_dict(self, info, full_title=None, secret_token=None):
         track_id = compat_str(info['id'])
         title = info['title']
@@ -389,7 +490,7 @@ def add_format(f, protocol, is_preview=False):
             if not format_url:
                 continue
             stream = self._download_json(
-                format_url, track_id, query=query, fatal=False)
+                format_url, track_id, query=query, fatal=False, headers=self._HEADERS)
             if not isinstance(stream, dict):
                 continue
             stream_url = url_or_none(stream.get('url'))
@@ -423,8 +524,8 @@ def add_format(f, protocol, is_preview=False):
 
         thumbnails = []
         artwork_url = info.get('artwork_url')
-        thumbnail = artwork_url or user.get('avatar_url')
-        if isinstance(thumbnail, compat_str):
+        thumbnail = url_or_none(artwork_url or user.get('avatar_url'))
+        if thumbnail:
             if re.search(self._IMAGE_REPL_RE, thumbnail):
                 for image_id, size in self._ARTWORK_MAP.items():
                     i = {
@@ -487,12 +588,12 @@ def _real_extract(self, url):
             info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
 
         info = self._download_json(
-            info_json_url, full_title, 'Downloading info JSON', query=query)
+            info_json_url, full_title, 'Downloading info JSON', query=query, headers=self._HEADERS)
 
         return self._extract_info_dict(info, full_title, token)
 
 
-class SoundcloudPlaylistBaseIE(SoundcloudIE):
+class SoundcloudPlaylistBaseIE(SoundcloudBaseIE):
     def _extract_set(self, playlist, token=None):
         playlist_id = compat_str(playlist['id'])
         tracks = playlist.get('tracks') or []
@@ -503,7 +604,7 @@ def _extract_set(self, playlist, token=None):
                     'ids': ','.join([compat_str(t['id']) for t in tracks]),
                     'playlistId': playlist_id,
                     'playlistSecretToken': token,
-                })
+                }, headers=self._HEADERS)
         entries = []
         for track in tracks:
             track_id = str_or_none(track.get('id'))
@@ -523,7 +624,7 @@ def _extract_set(self, playlist, token=None):
 
 
 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[:\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
@@ -536,6 +637,15 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
     }, {
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
         'only_matching': True,
+    }, {
+        'url': 'https://soundcloud.com/discover/sets/weekly::flacmatic',
+        'only_matching': True,
+    }, {
+        'url': 'https://soundcloud.com/discover/sets/charts-top:all-music:de',
+        'only_matching': True,
+    }, {
+        'url': 'https://soundcloud.com/discover/sets/charts-top:hiphoprap:kr',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -547,7 +657,7 @@ def _real_extract(self, url):
             full_title += '/' + token
 
         info = self._download_json(self._resolv_url(
-            self._BASE_URL + full_title), full_title)
+            self._BASE_URL + full_title), full_title, headers=self._HEADERS)
 
         if 'errors' in info:
             msgs = (compat_str(err['error_message']) for err in info['errors'])
@@ -556,66 +666,65 @@ def _real_extract(self, url):
         return self._extract_set(info, token)
 
 
-class SoundcloudPagedPlaylistBaseIE(SoundcloudIE):
+class SoundcloudPagedPlaylistBaseIE(SoundcloudBaseIE):
     def _extract_playlist(self, base_url, playlist_id, playlist_title):
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': playlist_title,
+            'entries': self._entries(base_url, playlist_id),
+        }
+
+    def _entries(self, url, playlist_id):
         # Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200.
         # https://developers.soundcloud.com/blog/offset-pagination-deprecated
         COMMON_QUERY = {
             'limit': 200,
             'linked_partitioning': '1',
+            'offset': 0,
         }
 
         query = COMMON_QUERY.copy()
-        query['offset'] = 0
-
-        next_href = base_url
-
-        entries = []
-        for i in itertools.count():
-            response = self._download_json(
-                next_href, playlist_id,
-                'Downloading track page %s' % (i + 1), query=query)
-
-            collection = response['collection']
-
-            if not isinstance(collection, list):
-                collection = []
-
-            # Empty collection may be returned, in this case we proceed
-            # straight to next_href
+        retries = 3
 
-            def resolve_entry(candidates):
-                for cand in candidates:
-                    if not isinstance(cand, dict):
-                        continue
-                    permalink_url = url_or_none(cand.get('permalink_url'))
-                    if not permalink_url:
-                        continue
+        def resolve_entry(*candidates):
+            for cand in candidates:
+                if not isinstance(cand, dict):
+                    continue
+                permalink_url = url_or_none(cand.get('permalink_url'))
+                if permalink_url:
                     return self.url_result(
                         permalink_url,
                         SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
                         str_or_none(cand.get('id')), cand.get('title'))
 
-            for e in collection:
-                entry = resolve_entry((e, e.get('track'), e.get('playlist')))
-                if entry:
-                    entries.append(entry)
-
-            next_href = response.get('next_href')
-            if not next_href:
+        for i in itertools.count():
+            last_error = None
+            for attempt in range(0, retries + 1):
+                if last_error:
+                    self._downloader.report_warning('%s. Retrying ...' % remove_end(last_error, '.'), playlist_id)
+                try:
+                    response = self._download_json(
+                        url, playlist_id, query=query, headers=self._HEADERS,
+                        note='Downloading track page %s%s' % (i + 1, ' (retry #%d)' % (attempt, ) if attempt > 0 else ''))
+                    if not isinstance(response, dict):
+                        response = {}
+                    break
+                except ExtractorError as e:
+                    # Downloading page may result in intermittent 502 HTTP error
+                    # See https://github.com/yt-dlp/yt-dlp/issues/872
+                    if attempt >= retries or not isinstance(e.cause, compat_HTTPError) or e.cause.code != 502:
+                        raise
+                    last_error = compat_str(e.cause or e.msg)
+
+            for e in try_get(response, lambda x: x['collection'], list) or []:
+                yield resolve_entry(e, e.get('track'), e.get('playlist'))
+
+            url = url_or_none(response.get('next_href'))
+            if not url:
                 break
 
-            next_href = response['next_href']
-            parsed_next_href = compat_urlparse.urlparse(next_href)
-            query = compat_urlparse.parse_qs(parsed_next_href.query)
-            query.update(COMMON_QUERY)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': playlist_title,
-            'entries': entries,
-        }
+            query.pop('offset', None)
 
 
 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
@@ -696,7 +805,7 @@ def _real_extract(self, url):
 
         user = self._download_json(
             self._resolv_url(self._BASE_URL + uploader),
-            uploader, 'Downloading user info')
+            uploader, 'Downloading user info', headers=self._HEADERS)
 
         resource = mobj.group('rsrc') or 'all'
 
@@ -721,7 +830,7 @@ class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
     def _real_extract(self, url):
         track_name = self._match_id(url)
 
-        track = self._download_json(self._resolv_url(url), track_name)
+        track = self._download_json(self._resolv_url(url), track_name, headers=self._HEADERS)
         track_id = self._search_regex(
             r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
 
@@ -730,6 +839,54 @@ def _real_extract(self, url):
             track_id, 'Track station: %s' % track['title'])
 
 
+class SoundcloudRelatedIE(SoundcloudPagedPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<slug>[\w\d-]+/[\w\d-]+)/(?P<relation>albums|sets|recommended)'
+    IE_NAME = 'soundcloud:related'
+    _TESTS = [{
+        'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/recommended',
+        'info_dict': {
+            'id': '1084577272',
+            'title': 'Sexapil - Pingers 5 (Recommended)',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/albums',
+        'info_dict': {
+            'id': '1084577272',
+            'title': 'Sexapil - Pingers 5 (Albums)',
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/sets',
+        'info_dict': {
+            'id': '1084577272',
+            'title': 'Sexapil - Pingers 5 (Sets)',
+        },
+        'playlist_mincount': 4,
+    }]
+
+    _BASE_URL_MAP = {
+        'albums': 'tracks/%s/albums',
+        'sets': 'tracks/%s/playlists_without_albums',
+        'recommended': 'tracks/%s/related',
+    }
+
+    def _real_extract(self, url):
+        slug, relation = re.match(self._VALID_URL, url).group('slug', 'relation')
+
+        track = self._download_json(
+            self._resolv_url(self._BASE_URL + slug),
+            slug, 'Downloading track info', headers=self._HEADERS)
+
+        if track.get('errors'):
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, ','.join(
+                compat_str(err['error_message']) for err in track['errors'])), expected=True)
+
+        return self._extract_playlist(
+            self._API_V2_BASE + self._BASE_URL_MAP[relation] % track['id'], compat_str(track['id']),
+            '%s (%s)' % (track.get('title') or slug, relation.capitalize()))
+
+
 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
@@ -754,23 +911,24 @@ def _real_extract(self, url):
 
         data = self._download_json(
             self._API_V2_BASE + 'playlists/' + playlist_id,
-            playlist_id, 'Downloading playlist', query=query)
+            playlist_id, 'Downloading playlist', query=query, headers=self._HEADERS)
 
         return self._extract_set(data, token)
 
 
-class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
+class SoundcloudSearchIE(SoundcloudBaseIE, SearchInfoExtractor):
     IE_NAME = 'soundcloud:search'
     IE_DESC = 'Soundcloud search'
-    _MAX_RESULTS = float('inf')
     _TESTS = [{
         'url': 'scsearch15:post-avant jazzcore',
         'info_dict': {
+            'id': 'post-avant jazzcore',
             'title': 'post-avant jazzcore',
         },
         'playlist_count': 15,
     }]
 
+    _MAX_RESULTS = float('inf')
     _SEARCH_KEY = 'scsearch'
     _MAX_RESULTS_PER_PAGE = 200
     _DEFAULT_RESULTS_PER_PAGE = 50
@@ -786,30 +944,20 @@ def _get_collection(self, endpoint, collection_id, **query):
         })
         next_url = update_url_query(self._API_V2_BASE + endpoint, query)
 
-        collected_results = 0
-
         for i in itertools.count(1):
             response = self._download_json(
                 next_url, collection_id, 'Downloading page {0}'.format(i),
                 'Unable to download API page')
 
-            collection = response.get('collection', [])
-            if not collection:
-                break
-
-            collection = list(filter(bool, collection))
-            collected_results += len(collection)
-
-            for item in collection:
-                yield self.url_result(item['uri'], SoundcloudIE.ie_key())
-
-            if not collection or collected_results >= limit:
-                break
+            for item in try_get(response, lambda x: x['collection'], list) or []:
+                if item:
+                    yield self.url_result(item['uri'], SoundcloudIE.ie_key())
 
             next_url = response.get('next_href')
             if not next_url:
                 break
 
     def _get_n_results(self, query, n):
-        tracks = self._get_collection('search/tracks', query, limit=n, q=query)
-        return self.playlist_result(tracks, playlist_title=query)
+        return self.playlist_result(itertools.islice(
+            self._get_collection('search/tracks', query, limit=n, q=query),
+            0, None if n == float('inf') else n), query, query)

From 5fb5cd844805162c81b5cbb04ca2d18746a0a59d Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 3 Jun 2022 14:22:16 +0200
Subject: [PATCH 247/394] cleaning

---
 CONTRIBUTING.md        | 10 ++++++----
 docs/supportedsites.md |  1 +
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ff40cef784f..c86c5cb8b28 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,10 +150,12 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note:
+    * the test names use the extractor class name **without the trailing `IE`**
+    * tests with `only_matching` key in test's dict are not counted.
+8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9b2ac3766ef..9aa278a59a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -881,6 +881,7 @@
  - **SonyLIV**
  - **soundcloud**
  - **soundcloud:playlist**
+ - **soundcloud:related**
  - **soundcloud:search**: Soundcloud search
  - **soundcloud:set**
  - **soundcloud:trackstation**

From 6367b516bc7113ac028e1fd42001065a2b8b7e96 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 3 Jun 2022 14:23:28 +0200
Subject: [PATCH 248/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 309692e2d0b..c31e5a1a26f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.05.26'
+__version__ = '2022.06.03'

From e3e6b67f2ab3a3e37e97df5d057fda297dee4289 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 3 Jun 2022 14:24:33 +0200
Subject: [PATCH 249/394] cleaning

---
 CONTRIBUTING.md        | 10 ++++++----
 docs/supportedsites.md |  3 ++-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ff40cef784f..c86c5cb8b28 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,10 +150,12 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note:
+    * the test names use the extractor class name **without the trailing `IE`**
+    * tests with `only_matching` key in test's dict are not counted.
+8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0d3d31b089d..42b8aec15a9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -494,7 +494,8 @@
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
  - **m6**
- - **MacProVideo.com**
+ - **MacProVideo**
+ - **MacProVideoCourse**
  - **mailru**: Видео@Mail.Ru
  - **mailru:music**: Музыка@Mail.Ru
  - **mailru:music:search**: Музыка@Mail.Ru

From b5daee9b1743649345ec68a2e23d6ef5119d3a66 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 3 Jun 2022 16:06:54 +0100
Subject: [PATCH 250/394] [common] Improve parsing of jwplayer data

* don't crash parser if jwplayer_data is invalid

* use `label` in `sources[n]` as `format_id`

* relax `jwplayer().setup(...)` RE
---
 youtube_dl/extractor/common.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 797c35fd52d..0ea94525f5c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -70,6 +70,7 @@
     str_or_none,
     str_to_int,
     strip_or_none,
+    try_get,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -2713,7 +2714,7 @@ def manifest_url(manifest):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
+            r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>[^)]+)\s*\)''',
             webpage)
         if mobj:
             try:
@@ -2734,6 +2735,10 @@ def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
 
     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        if try_get(jwplayer_data, lambda x: x.get('playlist') or True) is None:
+            # not a dict
+            return {}
+
         # JWPlayer backward compatibility: flattened playlists
         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
         if 'playlist' not in jwplayer_data:
@@ -2833,12 +2838,13 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                     'ext': ext,
                 })
             else:
+                format_id = str_or_none(source.get('label'))
                 height = int_or_none(source.get('height'))
-                if height is None:
+                if height is None and format_id:
                     # Often no height is provided but there is a label in
                     # format like "1080p", "720p SD", or 1080.
                     height = int_or_none(self._search_regex(
-                        r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
+                        r'^(\d{3,4})[pP]?(?:\b|$)', format_id,
                         'height', default=None))
                 a_format = {
                     'url': source_url,
@@ -2847,6 +2853,9 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                     'tbr': int_or_none(source.get('bitrate')),
                     'ext': ext,
                 }
+                if format_id:
+                    a_format['format_id'] = format_id
+
                 if source_url.startswith('rtmp'):
                     a_format['ext'] = 'flv'
                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as

From ca85a5a2adbcc2b6810c7f9ca2d11e9f96fee482 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Tue, 20 Jul 2021 16:15:58 +0900
Subject: [PATCH 251/394] [common] Detect more manifest formats in
 _parse_jwplayer_formats() (closes #29596)"

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0ea94525f5c..0da1f83cce4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2819,11 +2819,11 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
             urls.append(source_url)
             source_type = source.get('type') or ''
             ext = mimetype2ext(source_type) or determine_ext(source_url)
-            if source_type == 'hls' or ext == 'm3u8':
+            if source_type == 'hls' or ext == 'm3u8' or 'format=m3u8-aapl' in source_url:
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id=m3u8_id, fatal=False))
-            elif source_type == 'dash' or ext == 'mpd':
+            elif source_type == 'dash' or ext == 'mpd' or 'format=mpd-time-csf' in source_url:
                 formats.extend(self._extract_mpd_formats(
                     source_url, video_id, mpd_id=mpd_id, fatal=False))
             elif ext == 'smil':

From 3687a4f9bdb025855acd2a4e782748c8e3810765 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Jun 2022 08:07:54 +0100
Subject: [PATCH 252/394] [DoodStream] Add extractor from yt-dlp back-port and
 improve

---
 youtube_dl/extractor/doodstream.py | 119 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 120 insertions(+)
 create mode 100644 youtube_dl/extractor/doodstream.py

diff --git a/youtube_dl/extractor/doodstream.py b/youtube_dl/extractor/doodstream.py
new file mode 100644
index 00000000000..70bf4e51207
--- /dev/null
+++ b/youtube_dl/extractor/doodstream.py
@@ -0,0 +1,119 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+import string
+import time
+
+from ..compat import compat_filter as filter
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    get_element_by_class,
+    parse_duration,
+    parse_filesize,
+    update_url_query,
+    unified_strdate,
+    url_or_none,
+)
+
+from .common import InfoExtractor
+
+
+class DoodStreamIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dood\.(?:to|watch)/[ed]/(?P<id>[a-z0-9]+)'
+    _TESTS = [{
+        'url': 'http://dood.to/e/5s1wmbdacezb',
+        'md5': '4568b83b31e13242b3f1ff96c55f0595',
+        'info_dict': {
+            'id': '5s1wmbdacezb',
+            'ext': 'mp4',
+            'title': 'Kat Wonders - Monthly May 2020',
+            'description': 'Kat Wonders - Monthly May 2020 | DoodStream.com',
+            'thumbnail': 'https://img.doodcdn.com/snaps/flyus84qgl2fsk4g.jpg',
+        },
+        'skip': 'Video not found',
+    }, {
+        'url': 'http://dood.watch/d/5s1wmbdacezb',
+        'md5': '4568b83b31e13242b3f1ff96c55f0595',
+        'info_dict': {
+            'id': '5s1wmbdacezb',
+            'ext': 'mp4',
+            'title': 'Kat Wonders - Monthly May 2020',
+            'description': 'Kat Wonders - Monthly May 2020 | DoodStream.com',
+            'thumbnail': 'https://img.doodcdn.com/snaps/flyus84qgl2fsk4g.jpg',
+        },
+        'skip': 'Video not found',
+    }, {
+        'url': 'https://dood.to/d/jzrxn12t2s7n',
+        'md5': '3207e199426eca7c2aa23c2872e6728a',
+        'info_dict': {
+            'id': 'jzrxn12t2s7n',
+            'ext': 'mp4',
+            'title': 'Stacy Cruz Cute ALLWAYSWELL',
+            'description': 'Stacy Cruz Cute ALLWAYSWELL | DoodStream.com',
+            'thumbnail': 'https://img.doodcdn.com/snaps/8edqd5nppkac3x8u.jpg',
+        },
+        'skip': 'Video not found',
+    }, {
+        'url': 'https://dood.to/d/is34uy8wvaet',
+        'md5': '04740d3ba93bcd638aa7a097d9226710',
+        'info_dict': {
+            'id': 'is34uy8wvaet',
+            'ext': 'mp4',
+            'title': 'Akhanda (2021) Telugu DVDScr MP3 700MB - DoodStream',
+            'upload_date': '20211202',
+            'filesize_approx': int,
+            'duration': 9886,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url = 'https://dood.to/e/' + video_id
+        headers = {
+            'User-Agent': 'Mozilla/5.0',  # (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/66.0',
+        }
+        webpage = self._download_webpage(url, video_id, headers=headers)
+
+        title = self._html_search_meta(('og:title', 'twitter:title'), webpage, default=None)
+        if not title:
+            title = self._html_search_regex(r'<title\b[^>]*>([^<]+?)(?:\|\s+DoodStream\s*)?</title', webpage, 'title')
+            if title == 'Video not found':
+                raise ExtractorError(title, expected=True)
+        token = self._html_search_regex(r'''[?&]token=([a-z0-9]+)[&']''', webpage, 'token')
+
+        headers.update({
+            # 'User-Agent': 'Mozilla/5.0',  # (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/66.0',
+            'referer': url
+        })
+
+        pass_md5 = self._html_search_regex(r'(/pass_md5.*?)\'', webpage, 'pass_md5')
+        final_url = (
+            self._download_webpage('https://dood.to' + pass_md5, video_id, headers=headers, note='Downloading final URL')
+            + ''.join((random.choice(string.ascii_letters + string.digits) for _ in range(10)))
+        )
+        final_url = update_url_query(final_url, {'token': token, 'expiry': int(time.time() * 1000), })
+
+        thumb = next(filter(None, (url_or_none(self._html_search_meta(x, webpage, default=None))
+                                   for x in ('og:image', 'twitter:image'))), None)
+        description = self._html_search_meta(
+            ('og:description', 'description', 'twitter:description'), webpage, default=None)
+
+        webpage = self._download_webpage('https://dood.to/d/' + video_id, video_id, headers=headers, fatal=False)
+
+        def get_class_text(x):
+            return clean_html(get_element_by_class(x, webpage))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': final_url,
+            'http_headers': headers,
+            'ext': 'mp4',
+            'upload_date': unified_strdate(get_class_text('uploadate')),
+            'duration': parse_duration(get_class_text('length')),
+            'filesize_approx': parse_filesize(get_class_text('size')),
+            'description': description,
+            'thumbnail': thumb,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..3d1d9375eca 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -299,6 +299,7 @@
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE
+from .doodstream import DoodStreamIE
 from .dotsub import DotsubIE
 from .douyutv import (
     DouyuShowIE,

From 9dd91758aea6b3427662cac1efde1c5b5c1de6c3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Jun 2022 09:26:41 +0100
Subject: [PATCH 253/394] [DoodStream] Support more TLDs * also fix title
 extraction

---
 youtube_dl/extractor/doodstream.py | 35 +++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/doodstream.py b/youtube_dl/extractor/doodstream.py
index 70bf4e51207..94411ebb053 100644
--- a/youtube_dl/extractor/doodstream.py
+++ b/youtube_dl/extractor/doodstream.py
@@ -21,7 +21,7 @@
 
 
 class DoodStreamIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dood\.(?:to|watch)/[ed]/(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?dood\.(?:to|watch|so|la|pm|sh|ws|one)/[ed]/(?P<id>[a-z0-9]+)'
     _TESTS = [{
         'url': 'http://dood.to/e/5s1wmbdacezb',
         'md5': '4568b83b31e13242b3f1ff96c55f0595',
@@ -61,11 +61,24 @@ class DoodStreamIE(InfoExtractor):
         'info_dict': {
             'id': 'is34uy8wvaet',
             'ext': 'mp4',
-            'title': 'Akhanda (2021) Telugu DVDScr MP3 700MB - DoodStream',
+            'title': 'Akhanda (2021) Telugu DVDScr MP3 700MB',
             'upload_date': '20211202',
+            'thumbnail': r're:https?://img\.doodcdn\.com?/[\w/]+\.jpg',
             'filesize_approx': int,
             'duration': 9886,
         }
+    }, {
+        'url': 'https://dood.so/d/wlihoael8uog',
+        'md5': '2c14444c89788cc309738c1560abe278',
+        'info_dict': {
+            'id': 'wlihoael8uog',
+            'ext': 'mp4',
+            'title': 'VID 20220319 161659',
+            'thumbnail': r're:https?://img\.doodcdn\.com?/splash/rmpnhb8ckkk79cge\.jpg',
+            'upload_date': '20220319',
+            'filesize_approx': int,
+            'duration': 12.0,
+        }
     }]
 
     def _real_extract(self, url):
@@ -76,11 +89,12 @@ def _real_extract(self, url):
         }
         webpage = self._download_webpage(url, video_id, headers=headers)
 
-        title = self._html_search_meta(('og:title', 'twitter:title'), webpage, default=None)
-        if not title:
-            title = self._html_search_regex(r'<title\b[^>]*>([^<]+?)(?:\|\s+DoodStream\s*)?</title', webpage, 'title')
-            if title == 'Video not found':
-                raise ExtractorError(title, expected=True)
+        def get_title(html, fatal=False):
+            return self._html_search_regex(r'<title\b[^>]*>([^<]+?)(?:[|-]\s+DoodStream\s*)?</title', html, 'title', fatal=fatal)
+
+        title = get_title(webpage)
+        if title == 'Video not found':
+            raise ExtractorError(title, expected=True)
         token = self._html_search_regex(r'''[?&]token=([a-z0-9]+)[&']''', webpage, 'token')
 
         headers.update({
@@ -100,7 +114,12 @@ def _real_extract(self, url):
         description = self._html_search_meta(
             ('og:description', 'description', 'twitter:description'), webpage, default=None)
 
-        webpage = self._download_webpage('https://dood.to/d/' + video_id, video_id, headers=headers, fatal=False)
+        webpage = self._download_webpage('https://dood.to/d/' + video_id, video_id, headers=headers, fatal=False) or ''
+
+        title = (
+            self._html_search_meta(('og:title', 'twitter:title'), webpage, default=None)
+            or get_title(webpage, fatal=(title is not None))
+            or title)
 
         def get_class_text(x):
             return clean_html(get_element_by_class(x, webpage))

From 30a954bad9831a8dc6320215df1a907d2d4db911 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jun 2022 14:18:46 +0100
Subject: [PATCH 254/394] [SpankBang] Back-port changes to SpankBangIE from
 yt-dlp * improve title extraction * add uploader_id * update test * but don't
 check file md5

---
 youtube_dl/extractor/spankbang.py | 32 ++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 37cb8c839e5..e33d753c53a 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -26,19 +26,24 @@ class SpankBangIE(InfoExtractor):
                         )
                     '''
     _TESTS = [{
-        'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
-        'md5': '1cc433e1d6aa14bc376535b8679302f7',
+        'url': 'https://spankbang.com/56b3d/video/the+slut+maker+hmv',
+        'md5': '5039ba9d26f6124a7fdea6df2f21e765',
         'info_dict': {
-            'id': '3vvn',
+            'id': '56b3d',
             'ext': 'mp4',
-            'title': 'fantasy solo',
-            'description': 'dillion harper masturbates on a bed',
+            'title': 'The Slut Maker HMV',
+            'description': 'Girls getting converted into cock slaves.',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'silly2587',
-            'timestamp': 1422571989,
-            'upload_date': '20150129',
+            'uploader': 'Mindself',
+            'uploader_id': 'mindself',
+            'timestamp': 1617109572,
+            'upload_date': '20210330',
             'age_limit': 18,
-        }
+        },
+        'params': {
+            # adaptive download
+            'skip_download': True,
+        },
     }, {
         # 480p only
         'url': 'http://spankbang.com/1vt0/video/solvane+gangbang',
@@ -134,15 +139,15 @@ def extract_format(format_id, format_url):
         info = self._search_json_ld(webpage, video_id, default={})
 
         title = self._html_search_regex(
-            r'(?s)<h1[^>]*>(.+?)</h1>', webpage, 'title', default=None)
+            r'(?s)<h1[^>]+\btitle=["\']([^"]+)["\']>', webpage, 'title', default=None)
         description = self._search_regex(
             r'<div[^>]+\bclass=["\']bottom[^>]+>\s*<p>[^<]*</p>\s*<p>([^<]+)',
             webpage, 'description', default=None)
         thumbnail = self._og_search_thumbnail(webpage, default=None)
         uploader = self._html_search_regex(
-            (r'(?s)<li[^>]+class=["\']profile[^>]+>(.+?)</a>',
-             r'class="user"[^>]*><img[^>]+>([^<]+)'),
-            webpage, 'uploader', default=None)
+            r'<svg[^>]+\bclass="(?:[^"]*?user[^"]*?)">.*?</svg>([^<]+)', webpage, 'uploader', default=None)
+        uploader_id = self._html_search_regex(
+            r'<a[^>]+href="/profile/([^"]+)"', webpage, 'uploader_id', default=None)
         duration = parse_duration(self._search_regex(
             r'<div[^>]+\bclass=["\']right_side[^>]+>\s*<span>([^<]+)',
             webpage, 'duration', default=None))
@@ -157,6 +162,7 @@ def extract_format(format_id, format_url):
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
+            'uploader_id': uploader_id,
             'duration': duration,
             'view_count': view_count,
             'formats': formats,

From 836463013c363d38a2e24dabe40b000df76079f1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jun 2022 14:23:54 +0100
Subject: [PATCH 255/394] [SpankBang] Rework SpankBangPlaylistIE with
 pagination

---
 youtube_dl/extractor/spankbang.py | 57 +++++++++++++++++++++++--------
 1 file changed, 43 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index e33d753c53a..b3bff8ba12b 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -1,11 +1,16 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    extract_attributes,
     ExtractorError,
+    get_element_by_class,
+    get_element_by_id,
     merge_dicts,
     parse_duration,
     parse_resolution,
@@ -173,32 +178,56 @@ def extract_format(format_id, format_url):
 
 class SpankBangPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[^/]+\.)?spankbang\.com/(?P<id>[\da-z]+)/playlist/(?P<display_id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://spankbang.com/ug0k/playlist/big+ass+titties',
         'info_dict': {
             'id': 'ug0k',
             'title': 'Big Ass Titties',
         },
-        'playlist_mincount': 40,
-    }
+        'playlist_mincount': 35,
+    }, {
+        # pagination required
+        'url': 'https://spankbang.com/51wxk/playlist/dance',
+        'info_dict': {
+            'id': '51wxk',
+            'title': 'Dance',
+        },
+        'playlist_mincount': 60,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(
-            url, playlist_id, headers={'Cookie': 'country=US; mobile=on'})
-
-        entries = [self.url_result(
-            urljoin(url, mobj.group('path')),
-            ie=SpankBangIE.ie_key(), video_id=mobj.group('id'))
-            for mobj in re.finditer(
-                r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/%s(?:(?!\1).)*)\1'
-                % re.escape(display_id), webpage)]
+        webpage = self._download_webpage(url, playlist_id)
+
+        def _entries(url, webpage=None):
+            for ii in itertools.count(1):
+                if not webpage:
+                    webpage = self._download_webpage(
+                        url, playlist_id,
+                        note='Downloading playlist page %d' % (ii, ),
+                        fatal=False)
+                if not webpage:
+                    break
+                # search <main id="container">...</main>.innerHTML
+                for mobj in re.finditer(
+                        r'''<a\b[^>]*?\bclass\s*=\s*('|")(?:(?:(?!\1).)+?\s)?\s*thumb\b[^>]*>''',
+                        get_element_by_id('container', webpage) or webpage):
+                    item_url = extract_attributes(mobj.group(0)).get('href')
+                    if item_url:
+                        yield urljoin(url, item_url)
+                next_url = self._search_regex(
+                    r'''\bhref\s*=\s*(["'])(?P<path>(?!\1).+?)/?\1''',
+                    get_element_by_class('next', webpage) or '',
+                    'continuation page', group='path', default=None)
+                if next_url is None or next_url in url:
+                    break
+                url, webpage = urljoin(url, next_url + '/'), None
 
         title = self._html_search_regex(
             r'<h1>([^<]+)\s+playlist\s*<', webpage, 'playlist title',
-            fatal=False)
+            fatal=False) or re.sub(r'(\w)\+(\w)', r'\1 \2', display_id).title()
 
-        return self.playlist_result(entries, playlist_id, title)
+        return self.playlist_from_matches(_entries(url, webpage), playlist_id, title, ie=SpankBangIE.ie_key())

From c6001d56b2d02d5d470a744f052f49c3a7f7b529 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 7 Jun 2022 12:17:43 +0100
Subject: [PATCH 256/394] [ITV] Apply default vanilla UA to avoid site blocking

* `Mozilla/5.0` avoids read timeout on extraction and 403 on d/l
* also detect and report `Episode not found` instead of generic 404
---
 youtube_dl/extractor/itv.py | 44 ++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 67a000e04b0..5abbf4ec74c 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -6,7 +6,11 @@
 from .common import InfoExtractor
 from .brightcove import BrightcoveNewIE
 
-from ..compat import compat_str
+from ..compat import (
+    compat_HTTPError,
+    compat_kwargs,
+    compat_str,
+)
 from ..utils import (  # noqa: F401
     base_url,
     clean_html,
@@ -25,7 +29,20 @@
 )
 
 
-class ITVIE(InfoExtractor):
+class ITVBaseIE(InfoExtractor):
+    # enforce default UA that ITV doesn't block
+    _VANILLA_UA = 'Mozilla/5.0'
+
+    def _request_webpage(self, *args, **kwargs):
+        headers = kwargs.get('headers', {})
+        if 'User-Agent' not in headers:
+            headers['User-Agent'] = self._VANILLA_UA
+            kwargs.update({'headers': headers, })
+            kwargs = compat_kwargs(kwargs)
+        return super(ITVBaseIE, self)._request_webpage(*args, **kwargs)
+
+
+class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
     _GEO_COUNTRIES = ['GB']
     _TESTS = [{
@@ -136,7 +153,18 @@ def _get_subtitles(self, video_id, variants, ios_playlist_url, headers, *args, *
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage, urlh = self._download_webpage_handle(url, video_id, expected_status=404)
+        title = (
+            self._html_search_meta(['og:title', 'twitter:title'], webpage)
+            or self._html_search(r'(?s)<title\b[^>]*>(.+?)(?:-\s+ITV\s+Hub\s*)?</title\b', 'title', webpage))
+        if any(sorry in title for sorry in ('Episode not available', "We're really sorry")):
+            raise ExtractorError(
+                '%s not found; %s said: %s' % (video_id, self.IE_NAME, title),
+                expected=True)
+        if urlh.getcode() == 404:
+            raise compat_HTTPError(
+                urlh.geturl(), 404, '%s (%s: %s)' % (urlh.msg or 'Not Found', video_id, title, ), urlh.headers, None)
+
         params = extract_attributes(self._search_regex(
             r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
         variants = self._parse_json(
@@ -154,7 +182,6 @@ def _real_extract(self, url):
 
         ios_playlist_url = params.get('data-video-playlist') or params['data-video-id']
         headers = self._generate_api_headers(params['data-video-hmac'])
-        headers['Referer'] = url
         ios_playlist = self._call_api(
             video_id, ios_playlist_url, headers, platform_tag_video, featureset_video)
 
@@ -214,20 +241,21 @@ def parse_duration(s):  # noqa: F811
             from re import sub
             from ..utils import parse_duration as utils_parse_duration
             return utils_parse_duration(
-                sub(r':(\d{3,})$', r'.\1', s or ''))
+                sub(r':(\d{3,})$', r'.\1', s or '') or None)
 
         return merge_dicts({
             'id': video_id,
-            'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
+            'title': title,
             'formats': formats,
             'subtitles': self.extract_subtitles(video_id, variants, ios_playlist_url, headers),
             'duration': parse_duration(video_data.get('Duration')),
             'description': clean_html(get_element_by_class('episode-info__synopsis', webpage)),
-            'thumbnails': thumbnails
+            'thumbnails': thumbnails,
+            'http_headers': {'User-Agent': self._VANILLA_UA, },
         }, info)
 
 
-class ITVBTCCIE(InfoExtractor):
+class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:news|btcc)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action',

From 14a087ec62d4a53ab6b0acbf0b560774927085e2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 7 Jun 2022 19:39:02 +0100
Subject: [PATCH 257/394] [SpankBang] Support category, tag, search, channel,
 star and profile pages

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/spankbang.py  | 171 ++++++++++++++++++++++++-----
 2 files changed, 142 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade71..8a5067dd902 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1153,6 +1153,7 @@
 from .spankbang import (
     SpankBangIE,
     SpankBangPlaylistIE,
+    SpankBangListIE,
 )
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE
diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index b3bff8ba12b..e76f9ead35a 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -5,6 +5,7 @@
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     extract_attributes,
@@ -183,6 +184,7 @@ class SpankBangPlaylistIE(InfoExtractor):
         'info_dict': {
             'id': 'ug0k',
             'title': 'Big Ass Titties',
+            'description': 'md5:65b01bb13a9276cf172a67a41304bafd',
         },
         'playlist_mincount': 35,
     }, {
@@ -191,43 +193,152 @@ class SpankBangPlaylistIE(InfoExtractor):
         'info_dict': {
             'id': '51wxk',
             'title': 'Dance',
+            'description': 'md5:7aae6991c65d561a9319ecab31f857e2',
         },
         'playlist_mincount': 60,
     }]
 
+    def _entries(self, url, playlist_id, webpage=None):
+        for ii in itertools.count(1):
+            if not webpage:
+                webpage = self._download_webpage(
+                    url, playlist_id,
+                    note='Downloading playlist page %d' % (ii, ),
+                    fatal=False)
+            if not webpage:
+                break
+            # search <main id="container">...</main>.innerHTML
+            for mobj in re.finditer(
+                    r'''<a\b[^>]*?\bclass\s*=\s*('|")(?:(?:(?!\1).)+?\s)?\s*thumb\b[^>]*>''',
+                    get_element_by_id('container', webpage) or webpage):
+                item_url = extract_attributes(mobj.group(0)).get('href')
+                if item_url:
+                    yield urljoin(url, item_url)
+            next_url = self._search_regex(
+                r'''\bhref\s*=\s*(["'])(?P<path>(?!\1).+?)/?\1''',
+                get_element_by_class('next', webpage) or '',
+                'continuation page', group='path', default=None)
+            if next_url is None or next_url in url:
+                break
+            url, webpage = urljoin(url, next_url), None
+            p_url = compat_urlparse.urlparse(url)
+            url = compat_urlparse.urlunparse(p_url._replace(path=p_url.path + '/'))
+
+    def _get_title(self, list_id, webpage, url):
+        return self._html_search_regex(
+            r'<h1>([^<]+)\s+playlist\s*<', webpage, 'playlist title',
+            fatal=False) or re.sub(r'(\w)\+(\w)', r'\1 \2', list_id).title()
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        mobj = re.match(self._VALID_URL, url).groupdict()
+        playlist_id = mobj['id']
+        display_id = mobj.get('display_id') or playlist_id
+        related = mobj.get('related')
 
         webpage = self._download_webpage(url, playlist_id)
 
-        def _entries(url, webpage=None):
-            for ii in itertools.count(1):
-                if not webpage:
-                    webpage = self._download_webpage(
-                        url, playlist_id,
-                        note='Downloading playlist page %d' % (ii, ),
-                        fatal=False)
-                if not webpage:
-                    break
-                # search <main id="container">...</main>.innerHTML
-                for mobj in re.finditer(
-                        r'''<a\b[^>]*?\bclass\s*=\s*('|")(?:(?:(?!\1).)+?\s)?\s*thumb\b[^>]*>''',
-                        get_element_by_id('container', webpage) or webpage):
-                    item_url = extract_attributes(mobj.group(0)).get('href')
-                    if item_url:
-                        yield urljoin(url, item_url)
-                next_url = self._search_regex(
-                    r'''\bhref\s*=\s*(["'])(?P<path>(?!\1).+?)/?\1''',
-                    get_element_by_class('next', webpage) or '',
-                    'continuation page', group='path', default=None)
-                if next_url is None or next_url in url:
-                    break
-                url, webpage = urljoin(url, next_url + '/'), None
+        def get_title():
+            t = self._get_title(display_id, webpage, url)
+            if related:
+                t = '%s of %s' % (related.title(), t, )
+            return t
 
-        title = self._html_search_regex(
-            r'<h1>([^<]+)\s+playlist\s*<', webpage, 'playlist title',
-            fatal=False) or re.sub(r'(\w)\+(\w)', r'\1 \2', display_id).title()
+        result = self.playlist_from_matches(self._entries(url, playlist_id, webpage), playlist_id, get_title(), ie=SpankBangIE.ie_key())
+        description = self._html_search_meta(('description', 'og:description'), webpage)
+        if description:
+            result['description'] = description
+        return result
+
+
+class SpankBangListIE(SpankBangPlaylistIE):
+    _VALID_URL = r'''(?x)
+            https?://(?:[^/]+\.)?spankbang\.com/
+                (?:
+                    category|tag|s|
+                    [^/]+/channel|
+                    [^/]+/pornstar|
+                    (?P<profile>profile)
+                )/(?P<id>[^/?#]+)(?(profile)/(?P<related>videos|likes|playlists)?|)'''
+    _TESTS = [{
+        'url': 'https://spankbang.com/category/striptease/?p=d',
+        'info_dict': {
+            'id': 'striptease',
+            'title': 'Category: Striptease Porn (Jun 2022) - Joi & Strip (today)',
+            'description': 'md5:d4580b5fc1e8fae9f627178964989959',
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://spankbang.com/tag/monty/',
+        'info_dict': {
+            'id': 'monty',
+            'title': 'Tag: Monty Porn - Jenni Lee & British',
+            'description': 'md5:93535d53455f10b934034bfb9562de35',
+        },
+        'playlist_mincount': 40,
+    }, {
+        'url': 'https://spankbang.com/s/silly/?q=fhd&d=10',
+        'info_dict': {
+            'id': 'silly',
+            'title': 'Search: Silly HD Porn - Chubby Teens & Cum Swallow (FHD, 10+ mins)',
+            'description': 'md5:a241b24847f1efd7cc3cfc4ef2b22429',
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'https://spankbang.com/er/channel/purexxxfilms/?o=new',
+        'info_dict': {
+            'id': 'purexxxfilms',
+            'title': 'PureXXXFilms Channel (new)',
+            'description': 'md5:bb61fbf523511f3bd15aea0360bdbdc0',
+        },
+        'playlist_mincount': 65,
+    }, {
+        'url': 'https://spankbang.com/5pj/pornstar/promise/',
+        'info_dict': {
+            'id': 'promise',
+            'title': 'Promise Pornstar Page',
+            'description': 'md5:ff08a6ac2d6cd1225f0fae74ac896d63',
+        },
+        'playlist_mincount': 90,
+    }, {
+        'url': 'https://spankbang.com/profile/migouche/likes',
+        'info_dict': {
+            'id': 'migouche',
+            'title': 'Likes of migouche Profile',
+            'description': 'md5:b97540209879a62cadf2af6b0934bbc9',
+        },
+        'playlist_mincount': 2,
+    },
+    ]
 
-        return self.playlist_from_matches(_entries(url, webpage), playlist_id, title, ie=SpankBangIE.ie_key())
+    def _get_title(self, list_id, webpage, url):
+        title = (
+            self._og_search_title(webpage, default=None)
+            or self._html_search_regex(
+                r'<title\b[^>]*>([^<]+)</title', webpage, 'channel title',
+                fatal=False))
+        if title:
+            title = re.sub(r'\s*(?:&\s+(?:Porn\s+))?(?:Videos\s+)?[@-]\s*SpankBang\s*$', '', title)
+        title = title or re.sub(r'(\w)\+(\w)', r'\1 \2', list_id).title()
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        quals = []
+        for q in ('o', 'q', 'd', 'p'):
+            v = qs.get(q, [None])[-1]
+            if not v:
+                continue
+            if q == 'q':
+                v = v.upper()
+            elif q == 'd':
+                v += '+ mins'
+            elif q == 'p':
+                v = {'d': 'today', 'w': 'this week', 'm': 'this month', 'y': 'this year'}.get(v, v)
+            quals.append(v)
+        quals = ', '.join(quals)
+        if quals:
+            title += ' (%s)' % (quals, )
+        m = re.search(r'/(category|tag|s)/', url)
+        if m:
+            pfx = m.group(1)
+            if pfx == 's':
+                pfx = 'search'
+            title = '%s: %s' % (pfx.title(), title)
+        return title

From 79819f441beadb0dac3936c6b63464dca04c12da Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 9 Jun 2022 23:42:06 +0100
Subject: [PATCH 258/394] [BitChute] Back-port from yt-dlp and upgrade BitChute
 * extract timestamp instead of upload date * improve title and description
 extraction BitChuteChannel * fix listing entries Throughout * relax regexes

---
 youtube_dl/extractor/bitchute.py | 107 +++++++++++++++++++------------
 1 file changed, 67 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 0c773e66e1c..459602079f4 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -6,8 +6,15 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    ExtractorError,
+    GeoRestrictedError,
+    get_element_by_class,
+    get_element_by_id,
     orderedSet,
+    strip_or_none,
     unified_strdate,
+    unified_timestamp,
     urlencode_postdata,
 )
 
@@ -15,16 +22,17 @@
 class BitChuteIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)'
     _TESTS = [{
-        'url': 'https://www.bitchute.com/video/szoMrox2JEI/',
-        'md5': '66c4a70e6bfc40dcb6be3eb1d74939eb',
+        'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
+        'md5': '7e427d7ed7af5a75b5855705ec750e2b',
         'info_dict': {
-            'id': 'szoMrox2JEI',
+            'id': 'UGlrF9o9b-Q',
             'ext': 'mp4',
-            'title': 'Fuck bitches get money',
-            'description': 'md5:3f21f6fb5b1d17c3dee9cf6b5fe60b3a',
+            'title': 'This is the first video on #BitChute !',
+            'timestamp': 1483425420,
+            'upload_date': '20170103',
+            'description': 'md5:a0337e7b1fe39e32336974af8173a034',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Victoria X Rave',
-            'upload_date': '20170813',
+            'uploader': 'BitChute',
         },
     }, {
         'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
@@ -34,6 +42,13 @@ class BitChuteIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        urls = re.finditer(
+            r'''<(?:script|iframe)\b[^>]+\bsrc\s*=\s*(["'])(?P<url>%s)''' % (BitChuteIE._VALID_URL, ),
+            webpage)
+        return (mobj.group('url') for mobj in urls)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -42,43 +57,55 @@ def _real_extract(self, url):
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
             })
 
-        title = self._html_search_regex(
-            (r'<[^>]+\bid=["\']video-title[^>]+>([^<]+)', r'<title>([^<]+)'),
-            webpage, 'title', default=None) or self._html_search_meta(
-            'description', webpage, 'title',
-            default=None) or self._og_search_description(webpage)
+        title = (
+            self._og_search_title(webpage, default=None) 
+            or strip_or_none(clean_html(get_element_by_id('video-title', webpage)))
+            or self._html_search_regex(r'(?s)<title\b[^>]*>.*?</title', webpage, 'title'))
 
-        format_urls = []
-        for mobj in re.finditer(
-                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage):
-            format_urls.append(mobj.group('url'))
-        format_urls.extend(re.findall(r'as=(https?://[^&"\']+)', webpage))
+        format_urls = [
+            mobj.group('url')
+            for mobj in re.finditer(
+                r'''\baddWebSeed\s*\(\s*(["'])(?P<url>(?:(?!\1).)+)\1''', webpage)]
+        format_urls.extend(re.findall(r'''as=(https?://[^&"']+)''', webpage))
 
         formats = [
             {'url': format_url}
             for format_url in orderedSet(format_urls)]
 
         if not formats:
-            formats = self._parse_html5_media_entries(
-                url, webpage, video_id)[0]['formats']
+            entries = self._parse_html5_media_entries(
+                url, webpage, video_id)
+            if not entries:
+                error = strip_or_none(clean_html(self.get_element_by_id('video-title'))) or 'Cannot find video'
+                if error == 'Video Unavailable':
+                    raise GeoRestrictedError(error)
+                raise ExtractorError(error)
+            formats = entries[0]['formats']
 
         self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
-        description = self._html_search_regex(
-            r'(?s)<div\b[^>]+\bclass=["\']full hidden[^>]+>(.+?)</div>',
-            webpage, 'description', fatal=False)
-        thumbnail = self._og_search_thumbnail(
-            webpage, default=None) or self._html_search_meta(
-            'twitter:image:src', webpage, 'thumbnail')
+        description = (
+            self._og_search_description(webpage)
+            or clean_html(get_element_by_id('video-description', webpage))
+            or self._html_search_regex(
+                r'(?s)<div\b[^>]+\bclass=["\']full hidden[^>]+>(.+?)</div>',
+                webpage, 'description', fatal=False))
+        thumbnail = self._html_search_meta(
+            ('og:image', 'twitter:image:src'), webpage, 'thumbnail', fatal=False)
         uploader = self._html_search_regex(
-            (r'(?s)<div class=["\']channel-banner.*?<p\b[^>]+\bclass=["\']name[^>]+>(.+?)</p>',
-             r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>'),
+            (r'''(?s)<div\b[^>]+?\bclass\s*=\s*["']channel-banner.*?<p\b[^>]+\bclass\s*=\s*["']name\b[^>]+>(.+?)</p>''',
+             r'''(?s)<p\b[^>]+\bclass\s*=\s*["']video-author\b[^>]+>(.+?)</p>'''),
             webpage, 'uploader', fatal=False)
 
-        upload_date = unified_strdate(self._search_regex(
-            r'class=["\']video-publish-date[^>]+>[^<]+ at \d+:\d+ UTC on (.+?)\.',
-            webpage, 'upload date', fatal=False))
+        def more_unified_timestamp(x):
+            # ... at hh:mm TZ on month nth.
+            y = re.split(r'\s+at\s+', x or '')[-1]
+            y = re.sub(r'(?:^\s+|\s+$|\.+$|(?<=\d)(?:st|nd|rd|th))', '', y)
+            y = ' '.join(reversed(re.split(r'\s+on\s+', y, 1)))
+            return unified_timestamp(y) or unified_timestamp(x)
+
+        timestamp = more_unified_timestamp(get_element_by_class('video-publish-date', webpage))
 
         return {
             'id': video_id,
@@ -86,7 +113,7 @@ def _real_extract(self, url):
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'formats': formats,
         }
 
@@ -94,22 +121,22 @@ def _real_extract(self, url):
 class BitChuteChannelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/channel/(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'https://www.bitchute.com/channel/victoriaxrave/',
-        'playlist_mincount': 185,
+        'url': 'https://www.bitchute.com/channel/livesonnet/',
+        'playlist_mincount': 135,
         'info_dict': {
-            'id': 'victoriaxrave',
+            'id': 'livesonnet',
         },
     }
 
     _TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
 
     def _entries(self, channel_id):
-        channel_url = 'https://www.bitchute.com/channel/%s/' % channel_id
+        channel_url = 'https://www.bitchute.com/channel/%s' % (channel_id, )
         offset = 0
         for page_num in itertools.count(1):
             data = self._download_json(
-                '%sextend/' % channel_url, channel_id,
-                'Downloading channel page %d' % page_num,
+                channel_url + '/extend/', channel_id,
+                'Downloading channel page %d' % (page_num, ),
                 data=urlencode_postdata({
                     'csrfmiddlewaretoken': self._TOKEN,
                     'name': '',
@@ -118,7 +145,7 @@ def _entries(self, channel_id):
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                     'Referer': channel_url,
                     'X-Requested-With': 'XMLHttpRequest',
-                    'Cookie': 'csrftoken=%s' % self._TOKEN,
+                    'Cookie': 'csrftoken=' + self._TOKEN,
                 })
             if data.get('success') is False:
                 break
@@ -126,14 +153,14 @@ def _entries(self, channel_id):
             if not html:
                 break
             video_ids = re.findall(
-                r'class=["\']channel-videos-image-container[^>]+>\s*<a\b[^>]+\bhref=["\']/video/([^"\'/]+)',
+                r'''class\s*=\s*["']channel-videos-image-container[^>]+>\s*<a\b[^>]+\bhref\s*=\s*["']/video/([^"'/]+)''',
                 html)
             if not video_ids:
                 break
             offset += len(video_ids)
             for video_id in video_ids:
                 yield self.url_result(
-                    'https://www.bitchute.com/video/%s' % video_id,
+                    'https://www.bitchute.com/video/' + video_id,
                     ie=BitChuteIE.ie_key(), video_id=video_id)
 
     def _real_extract(self, url):

From 1f627925519156b4dde65e222f0dcc0cefc50532 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Jun 2022 01:31:53 +0100
Subject: [PATCH 259/394] [BitChute] Added BitChutePlaylistIE * closes #26725

---
 youtube_dl/extractor/bitchute.py   | 160 ++++++++++++++++++++---------
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 114 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 459602079f4..8946933b86e 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -11,15 +11,67 @@
     GeoRestrictedError,
     get_element_by_class,
     get_element_by_id,
+    merge_dicts,
     orderedSet,
     strip_or_none,
-    unified_strdate,
     unified_timestamp,
     urlencode_postdata,
 )
 
 
-class BitChuteIE(InfoExtractor):
+class BitChuteBaseIE(InfoExtractor):
+    _USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36'
+
+    def _list_entries(self, list_id):
+        TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
+        list_url = self._API_URL + list_id
+        offset = 0
+        for page_num in itertools.count(1):
+            data = self._download_json(
+                list_url + '/extend/', list_id,
+                'Downloading list page %d' % (page_num, ),
+                data=urlencode_postdata({
+                    'csrfmiddlewaretoken': TOKEN,
+                    'name': '',
+                    'offset': offset,
+                }), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                    'Referer': list_url,
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'Cookie': 'csrftoken=' + TOKEN,
+                })
+            if data.get('success') is False:
+                break
+            html = data.get('html')
+            if not html:
+                break
+            video_ids = re.findall(
+                r'''class\s*=\s*["'](?:channel-videos-)?image-container[^>]+>\s*<a\b[^>]+\bhref\s*=\s*["']/video/([^"'/]+)''',
+                html)
+            if not video_ids:
+                break
+            offset += len(video_ids)
+            for video_id in video_ids:
+                yield self.url_result(
+                    'https://www.bitchute.com/video/' + video_id,
+                    ie=BitChuteIE.ie_key(), video_id=video_id)
+
+    def _search_title(self, html, title_id, **kwargs):
+        return (
+            strip_or_none(clean_html(get_element_by_id(title_id, html)))
+            or self._og_search_title(html, default=None)
+            or self._html_search_regex(r'(?s)<title\b[^>]*>.*?</title', html, 'title', **kwargs))
+
+    def _search_description(self, html, descr_id):
+        return (
+            self._og_search_description(html)
+            or (descr_id and clean_html(get_element_by_id(descr_id, html)))
+            or self._html_search_regex(
+                r'(?s)<div\b[^>]+\bclass=["\']full hidden[^>]+>(.+?)</div>',
+                html, 'description', fatal=False))
+
+
+class BitChuteIE(BitChuteBaseIE):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
@@ -53,14 +105,12 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.bitchute.com/video/%s' % video_id, video_id, headers={
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
+            'https://www.bitchute.com/video/' + video_id, video_id,
+            headers={
+                'User-Agent': self._USER_AGENT,
             })
 
-        title = (
-            self._og_search_title(webpage, default=None) 
-            or strip_or_none(clean_html(get_element_by_id('video-title', webpage)))
-            or self._html_search_regex(r'(?s)<title\b[^>]*>.*?</title', webpage, 'title'))
+        title = self._search_title(webpage, 'video-title')
 
         format_urls = [
             mobj.group('url')
@@ -86,8 +136,7 @@ def _real_extract(self, url):
         self._sort_formats(formats)
 
         description = (
-            self._og_search_description(webpage)
-            or clean_html(get_element_by_id('video-description', webpage))
+            self._search_description(webpage, 'video-description')
             or self._html_search_regex(
                 r'(?s)<div\b[^>]+\bclass=["\']full hidden[^>]+>(.+?)</div>',
                 webpage, 'description', fatal=False))
@@ -118,52 +167,69 @@ def more_unified_timestamp(x):
         }
 
 
-class BitChuteChannelIE(InfoExtractor):
+class BitChuteChannelIE(BitChuteBaseIE):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/channel/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.bitchute.com/channel/livesonnet/',
         'playlist_mincount': 135,
         'info_dict': {
             'id': 'livesonnet',
+            'title': 'livesonnet_vidz',
+            'description': 'md5:b0017be20656a1347eeb84f1049fc424',
         },
     }
+    _API_URL = 'https://www.bitchute.com/channel/'
 
-    _TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
 
-    def _entries(self, channel_id):
-        channel_url = 'https://www.bitchute.com/channel/%s' % (channel_id, )
-        offset = 0
-        for page_num in itertools.count(1):
-            data = self._download_json(
-                channel_url + '/extend/', channel_id,
-                'Downloading channel page %d' % (page_num, ),
-                data=urlencode_postdata({
-                    'csrfmiddlewaretoken': self._TOKEN,
-                    'name': '',
-                    'offset': offset,
-                }), headers={
-                    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-                    'Referer': channel_url,
-                    'X-Requested-With': 'XMLHttpRequest',
-                    'Cookie': 'csrftoken=' + self._TOKEN,
-                })
-            if data.get('success') is False:
-                break
-            html = data.get('html')
-            if not html:
-                break
-            video_ids = re.findall(
-                r'''class\s*=\s*["']channel-videos-image-container[^>]+>\s*<a\b[^>]+\bhref\s*=\s*["']/video/([^"'/]+)''',
-                html)
-            if not video_ids:
-                break
-            offset += len(video_ids)
-            for video_id in video_ids:
-                yield self.url_result(
-                    'https://www.bitchute.com/video/' + video_id,
-                    ie=BitChuteIE.ie_key(), video_id=video_id)
+        webpage = self._download_webpage(
+            'https://www.bitchute.com/channel/' + channel_id, channel_id,
+            headers={
+                'User-Agent': self._USER_AGENT,
+            })
+
+        title = self._search_title(webpage, 'channel-title', default=None)
+
+        result = self.playlist_result(
+            self._list_entries(channel_id), playlist_id=channel_id, playlist_title=title)
+
+        return merge_dicts(
+            result,
+            {'description': self._search_description(webpage, 'channel-description')})
+
+
+class BitChutePlaylistIE(BitChuteBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?bitchute\.com/playlist/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://www.bitchute.com/playlist/g4WTfWTdYEQa/',
+        'playlist_mincount': 1,
+        'info_dict': {
+            'id': 'g4WTfWTdYEQa',
+            'title': 'Podcasts',
+            'description': 'Podcast Playlist',
+        },
+    }
+    _API_URL = 'https://www.bitchute.com/playlist/'
 
     def _real_extract(self, url):
-        channel_id = self._match_id(url)
-        return self.playlist_result(
-            self._entries(channel_id), playlist_id=channel_id)
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://www.bitchute.com/playlist/' + playlist_id, playlist_id,
+            headers={
+                'User-Agent': self._USER_AGENT,
+            })
+
+        title = self._search_title(webpage, 'playlist-title', default=None)
+
+        result = self.playlist_result(
+            self._list_entries(playlist_id), playlist_id=playlist_id, playlist_title=title)
+
+        description = (
+            strip_or_none(clean_html(get_element_by_class('description', webpage)))
+            or self._search_description(webpage, None))
+
+        return merge_dicts(
+            result,
+            {'description': description})
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..d82b1e15b12 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -132,6 +132,7 @@
 from .bitchute import (
     BitChuteIE,
     BitChuteChannelIE,
+    BitChutePlaylistIE,
 )
 from .biqle import BIQLEIE
 from .bleacherreport import (

From 5ed0057798f636782985e890b61aba2829188897 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Jun 2022 05:06:21 +0100
Subject: [PATCH 260/394] [BitChute] Handle NSFW in listings and single videos
 * listing: include NSFW if --age-limit 18 (or higher) * single video: extract
 'age_limit': 18 if NSFW Resolves #24419

---
 youtube_dl/extractor/bitchute.py | 75 +++++++++++++++++++++++++++-----
 1 file changed, 65 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 8946933b86e..0cc8d8fcf44 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -11,6 +11,7 @@
     GeoRestrictedError,
     get_element_by_class,
     get_element_by_id,
+    int_or_none,
     merge_dicts,
     orderedSet,
     strip_or_none,
@@ -26,6 +27,7 @@ def _list_entries(self, list_id):
         TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
         list_url = self._API_URL + list_id
         offset = 0
+        query = {'showall': '1'} if 18 <= int_or_none(self._downloader.params.get('age_limit')) or 0 else None
         for page_num in itertools.count(1):
             data = self._download_json(
                 list_url + '/extend/', list_id,
@@ -39,7 +41,7 @@ def _list_entries(self, list_id):
                     'Referer': list_url,
                     'X-Requested-With': 'XMLHttpRequest',
                     'Cookie': 'csrftoken=' + TOKEN,
-                })
+                }, query=query)
             if data.get('success') is False:
                 break
             html = data.get('html')
@@ -85,6 +87,22 @@ class BitChuteIE(BitChuteBaseIE):
             'description': 'md5:a0337e7b1fe39e32336974af8173a034',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'BitChute',
+            'age_limit': None,
+        },
+    }, {
+        # NSFW (#24419)
+        'url': 'https://www.bitchute.com/video/wrTrKp7PmFZC/',
+        'md5': '4ef880ce8d24e322172d41a0cf6f8096',
+        'info_dict': {
+            'id': 'wrTrKp7PmFZC',
+            'ext': 'mp4',
+            'title': "You Can't Stop Progress | Episode 2",
+            'timestamp': 1541476920,
+            'upload_date': '20181106',
+            'description': 'md5:f191b538a2c4d8f57540141a6bfd7eb0',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': "You Can't Stop Progress",
+            'age_limit': 18,
         },
     }, {
         'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
@@ -104,11 +122,20 @@ def _extract_urls(webpage):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
+        def get_error_title(html):
+            return strip_or_none(clean_html(get_element_by_class('page-title', html)))
+
+        def get_error_text(html):
+            return strip_or_none(clean_html(get_element_by_id('page-detail', html)))
+
+        webpage, urlh = self._download_webpage_handle(
             'https://www.bitchute.com/video/' + video_id, video_id,
             headers={
                 'User-Agent': self._USER_AGENT,
-            })
+            }, expected_status=404)
+
+        if urlh.getcode() == 404:
+            raise ExtractorError(get_error_title(webpage) or 'Cannot find video', expected=True)
 
         title = self._search_title(webpage, 'video-title')
 
@@ -126,10 +153,16 @@ def _real_extract(self, url):
             entries = self._parse_html5_media_entries(
                 url, webpage, video_id)
             if not entries:
-                error = strip_or_none(clean_html(self.get_element_by_id('video-title'))) or 'Cannot find video'
+                error = strip_or_none(clean_html(get_element_by_id('video-title', webpage)))
                 if error == 'Video Unavailable':
-                    raise GeoRestrictedError(error)
-                raise ExtractorError(error)
+                    raise GeoRestrictedError(error, expected=True)
+                error = get_error_title(webpage)
+                if error:
+                    reason = get_error_text(webpage)
+                    if reason:
+                        self.to_screen(reason)
+                    raise ExtractorError(error, expected=True)
+                raise ExtractorError('Cannot find video', )
             formats = entries[0]['formats']
 
         self._check_formats(formats, video_id)
@@ -164,20 +197,42 @@ def more_unified_timestamp(x):
             'uploader': uploader,
             'timestamp': timestamp,
             'formats': formats,
+            'age_limit': 18 if '>This video has been marked as Not Safe For Work' in webpage else None,
         }
 
 
 class BitChuteChannelIE(BitChuteBaseIE):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/channel/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.bitchute.com/channel/livesonnet/',
         'playlist_mincount': 135,
         'info_dict': {
             'id': 'livesonnet',
             'title': 'livesonnet_vidz',
             'description': 'md5:b0017be20656a1347eeb84f1049fc424',
+        }
+    }, {
+        # channel with NSFW content, not listed
+        'url': 'https://www.bitchute.com/channel/hQl9oMSgUyMX/',
+        'playlist_maxcount': 150,
+        'info_dict': {
+            'id': 'hQl9oMSgUyMX',
+            'title': "You Can't Stop Progress",
+            'description': 'md5:a7e3fd8cf02e96ddcc73e9f13d2ce768',
+        },
+    }, {
+        # channel with NSFW content, listed with adult age limit
+        'url': 'https://www.bitchute.com/channel/hQl9oMSgUyMX/',
+        'playlist_mincount': 160,
+        'info_dict': {
+            'id': 'hQl9oMSgUyMX',
+            'title': "You Can't Stop Progress",
+            'description': 'md5:a7e3fd8cf02e96ddcc73e9f13d2ce768',
         },
-    }
+        'params': {
+            'age_limit': 18,
+        }
+    }]
     _API_URL = 'https://www.bitchute.com/channel/'
 
     def _real_extract(self, url):
@@ -201,7 +256,7 @@ def _real_extract(self, url):
 
 class BitChutePlaylistIE(BitChuteBaseIE):
     _VALID_URL = r'https?://(?:www\.)?bitchute\.com/playlist/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.bitchute.com/playlist/g4WTfWTdYEQa/',
         'playlist_mincount': 1,
         'info_dict': {
@@ -209,7 +264,7 @@ class BitChutePlaylistIE(BitChuteBaseIE):
             'title': 'Podcasts',
             'description': 'Podcast Playlist',
         },
-    }
+    }]
     _API_URL = 'https://www.bitchute.com/playlist/'
 
     def _real_extract(self, url):

From add5e7dced00484343692dc04d568bc6b46ceeb3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Jun 2022 17:03:05 +0100
Subject: [PATCH 261/394] [BitChute] Extract `channel_url` * resolves #28500

---
 youtube_dl/extractor/bitchute.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 0cc8d8fcf44..533f555e32e 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -17,6 +17,7 @@
     strip_or_none,
     unified_timestamp,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -88,6 +89,7 @@ class BitChuteIE(BitChuteBaseIE):
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'BitChute',
             'age_limit': None,
+            'channel_url': 'https://www.bitchute.com/channel/bitchute/',
         },
     }, {
         # NSFW (#24419)
@@ -103,6 +105,7 @@ class BitChuteIE(BitChuteBaseIE):
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': "You Can't Stop Progress",
             'age_limit': 18,
+            'channel_url': 'https://www.bitchute.com/channel/ycsp/',
         },
     }, {
         'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
@@ -142,7 +145,7 @@ def get_error_text(html):
         format_urls = [
             mobj.group('url')
             for mobj in re.finditer(
-                r'''\baddWebSeed\s*\(\s*(["'])(?P<url>(?:(?!\1).)+)\1''', webpage)]
+                r'''\baddWebSeed\s*\(\s*("|')(?P<url>(?:(?!\1).)+)\1''', webpage)]
         format_urls.extend(re.findall(r'''as=(https?://[^&"']+)''', webpage))
 
         formats = [
@@ -189,6 +192,14 @@ def more_unified_timestamp(x):
 
         timestamp = more_unified_timestamp(get_element_by_class('video-publish-date', webpage))
 
+        # TODO: remove this work-around for class matching bug
+        webpage = re.split(r'''('|")channel-banner\1''', webpage, 1)[-1]
+        channel_details = get_element_by_class('details', webpage)
+        channel_details = channel_details and get_element_by_class('name', channel_details)
+        channel_url = urljoin(url, self._search_regex(
+            r'''<a\b[^>]*?\bhref\s*=\s*('|")(?P<url>(?:(?!\1).)+)''',
+            channel_details or '', 'channel url', group='url', default=None))
+
         return {
             'id': video_id,
             'title': title,
@@ -198,6 +209,7 @@ def more_unified_timestamp(x):
             'timestamp': timestamp,
             'formats': formats,
             'age_limit': 18 if '>This video has been marked as Not Safe For Work' in webpage else None,
+            'channel_url': channel_url,
         }
 
 

From 357b1940c347a286d01c0befe73d808db663c6a7 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 12 Jun 2022 18:31:21 +0200
Subject: [PATCH 262/394] cleaning

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 42b8aec15a9..34c67a6ea96 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -121,6 +121,7 @@
  - **BIQLE**
  - **BitChute**
  - **BitChuteChannel**
+ - **BitChutePlaylist**
  - **BleacherReport**
  - **BleacherReportCMS**
  - **Bloomberg**
@@ -250,6 +251,7 @@
  - **Disney**
  - **dlive:stream**
  - **dlive:vod**
+ - **DoodStream**
  - **Dotsub**
  - **DouyinUser**
  - **DouyinVideo**
@@ -385,6 +387,7 @@
  - **hotstar:playlist**
  - **Howcast**
  - **HowStuffWorks**
+ - **hrfernsehen**
  - **HRTi**
  - **HRTiPlaylist**
  - **Huajiao**: 花椒直播
@@ -897,6 +900,7 @@
  - **southpark.nl**
  - **southparkstudios.dk**
  - **SpankBang**
+ - **SpankBangList**
  - **SpankBangPlaylist**
  - **Spankwire**
  - **Spiegel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c31e5a1a26f..8a362a54db9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.06.03'
+__version__ = '2022.06.13'

From eeeb33c03edc3ca7d0589e98288e4126111a02c5 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 18:08:20 +0200
Subject: [PATCH 263/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 514d69a2a92..ec576fd87f2 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -87,7 +87,7 @@ def getrealurl(video_url, license_code):
 
 def getlicensetoken(license):
     modlicense = license.replace("$", "").replace("0", "1")
-    center = int(len(modlicense) / 2)
+    center = len(modlicense) // 2
     fronthalf = int(modlicense[:center + 1])
     backhalf = int(modlicense[center:])
 

From 9fd537dfcf00e9944147980ea7d885dff2ee4ce9 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 18:08:33 +0200
Subject: [PATCH 264/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index ec576fd87f2..2b53cb31394 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -46,8 +46,7 @@ def _real_extract(self, url):
         thumbnail = self._html_search_regex(r"preview_url:\s+'((?:https?:)?//media\.thisvid\.com/.+?\.jpg)',", webpage, 'thumbnail', fatal=False)
         uploader = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/[0-9]+/">(.+?)</a>', webpage, 'uploader')
         uploader_id = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/([0-9]+)/">.+?</a>', webpage, 'uploader_id')
-        if thumbnail.startswith("//"):
-            thumbnail = "https:" + thumbnail
+        thumbnail = sanitize_url(thumbnail)
         if (re.match(self._VALID_URL, url).group('type') == "videos"):
             display_id = main_id
         else:

From 08685ea6df7383daf3038db8debf1531bc7d9bb1 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 18:09:21 +0200
Subject: [PATCH 265/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 2b53cb31394..d79da29ce61 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -90,7 +90,11 @@ def getlicensetoken(license):
     fronthalf = int(modlicense[:center + 1])
     backhalf = int(modlicense[center:])
 
-    modlicense = str(4 * abs(fronthalf - backhalf))
+    modlicense = compat_str(4 * abs(fronthalf - backhalf))
+    retval = ""
+    for o in range(0, center + 1):
+        for i in range(1, 5):
+            retval += compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
     retval = ""
     for o in range(0, center + 1):
         for i in range(1, 5):

From 2b125142cec33e5afa4aea7c812aeca5d032b312 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 18:10:52 +0200
Subject: [PATCH 266/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d79da29ce61..d1068a11f51 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -50,7 +50,7 @@ def _real_extract(self, url):
         if (re.match(self._VALID_URL, url).group('type') == "videos"):
             display_id = main_id
         else:
-           display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False)
+            display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False)
 
         return {
             'id': video_id,

From d8c8d00dcc15afa8ad8c926a7d550abd72f6be88 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:26:46 +0200
Subject: [PATCH 267/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d1068a11f51..21e81a7a642 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -3,7 +3,12 @@
 import re
 
 from .common import InfoExtractor
-
+from ..compat import (
+    compat_str,
+)
+from ..utils import (
+    sanitize_url,
+)
 
 class ThisVidIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'

From d80d3d987bbf1bd9cd952e63d2bbc2b5e90f6880 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:27:12 +0200
Subject: [PATCH 268/394] Apply suggestions from code review

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 21e81a7a642..38ee2f4344e 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -35,7 +35,7 @@ class ThisVidIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        main_id = self._match_id(url)
+        main_id, type_ = re.match(self._VALID_URL, url).group('id', 'type')
         webpage = self._download_webpage(url, main_id)
 
         # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1

From 8ae8c8b4440c0bd47a1bda2ea85baf90d3184a65 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:27:21 +0200
Subject: [PATCH 269/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 38ee2f4344e..f344d1a6bd7 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -40,7 +40,7 @@ def _real_extract(self, url):
 
         # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1
         kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
-        if not kvs_version.startswith("5."):
+        if not (kvs_version and kvs_version.startswith('5.')):
             self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")
 
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')

From b55875373fbf1aa1e0ecd5c4b8b263de71711362 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:27:35 +0200
Subject: [PATCH 270/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index f344d1a6bd7..c56e8a6d5fa 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -96,12 +96,8 @@ def getlicensetoken(license):
     backhalf = int(modlicense[center:])
 
     modlicense = compat_str(4 * abs(fronthalf - backhalf))
-    retval = ""
+    retval = ''
     for o in range(0, center + 1):
         for i in range(1, 5):
             retval += compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
-    retval = ""
-    for o in range(0, center + 1):
-        for i in range(1, 5):
-            retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
     return retval

From ed67873cea4bd69c64869900fe49d77a5149ec44 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:27:57 +0200
Subject: [PATCH 271/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index c56e8a6d5fa..46e95f8ba70 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -73,7 +73,7 @@ def getrealurl(video_url, license_code):
     newmagic = urlparts[5][:32]
 
     for o in range(len(newmagic) - 1, -1, -1):
-        new = ""
+        new = ''
         l = (o + sum([int(n) for n in license[o:]])) % 32
 
         for i in range(0, len(newmagic)):

From eb4d2b93d1da857f4a2fac168381ad73a0870242 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:28:07 +0200
Subject: [PATCH 272/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 46e95f8ba70..1df84fa1c0a 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -77,12 +77,12 @@ def getrealurl(video_url, license_code):
         l = (o + sum([int(n) for n in license[o:]])) % 32
 
         for i in range(0, len(newmagic)):
-            if i == o:
-                new += newmagic[l]
-            elif i == l:
-                new += newmagic[o]
-            else:
-                new += newmagic[i]
+            idx = i
+            if idx == o:
+                idx = l
+            elif idx == l:
+                idx = o
+            new += newmagic[idx]
         newmagic = new
 
     urlparts[5] = newmagic + urlparts[5][32:]

From f04efe7a37820f464acd969645f9e38a2f2ce1a1 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:28:16 +0200
Subject: [PATCH 273/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 1df84fa1c0a..21c543bef91 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -52,10 +52,7 @@ def _real_extract(self, url):
         uploader = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/[0-9]+/">(.+?)</a>', webpage, 'uploader')
         uploader_id = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/([0-9]+)/">.+?</a>', webpage, 'uploader_id')
         thumbnail = sanitize_url(thumbnail)
-        if (re.match(self._VALID_URL, url).group('type') == "videos"):
-            display_id = main_id
-        else:
-            display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False)
+        display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False) if type_ == 'videos' else main_id
 
         return {
             'id': video_id,

From 14800d0617411f1b727c72585686b40c0813a927 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:28:24 +0200
Subject: [PATCH 274/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 21c543bef91..1d13f5dba5f 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -46,7 +46,7 @@ def _real_extract(self, url):
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
         # video_id, video_url and license_code from the 'flashvars' JSON object:
         video_id = self._html_search_regex(r"video_id:\s+'([0-9]+)',", webpage, 'video_id')
-        video_url = self._html_search_regex(r"video_url:\s+'(function/0/.+?)',", webpage, 'video_url')
+        video_url = self._html_search_regex(r"video_url:\s+'function/0/((?:[^/']+/){5,}.+?)',", webpage, 'video_url')
         license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')
         thumbnail = self._html_search_regex(r"preview_url:\s+'((?:https?:)?//media\.thisvid\.com/.+?\.jpg)',", webpage, 'thumbnail', fatal=False)
         uploader = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/[0-9]+/">(.+?)</a>', webpage, 'uploader')

From 3246d26950ba17c5af2546913fb4cf2f0df951e0 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:29:32 +0200
Subject: [PATCH 275/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 1d13f5dba5f..af698d8a47b 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -49,8 +49,14 @@ def _real_extract(self, url):
         video_url = self._html_search_regex(r"video_url:\s+'function/0/((?:[^/']+/){5,}.+?)',", webpage, 'video_url')
         license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')
         thumbnail = self._html_search_regex(r"preview_url:\s+'((?:https?:)?//media\.thisvid\.com/.+?\.jpg)',", webpage, 'thumbnail', fatal=False)
-        uploader = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/[0-9]+/">(.+?)</a>', webpage, 'uploader')
-        uploader_id = self._html_search_regex(r'<span>Added by: </span><a class="author" href="https://thisvid.com/members/([0-9]+)/">.+?</a>', webpage, 'uploader_id')
+        uploader = self._html_search_regex(r'''(?s)<span\b[^>]*>Added by:\s*</span><a\b[^>]+\bclass\s*=\s*["']author\b[^>]+\bhref\s*=\s*["']https://thisvid\.com/members/([0-9]+/.{3,}?)\s*</a>''', webpage, 'uploader', default='')
+        uploader = re.split(r'''/["'][^>]*>\s*''', uploader)
+        if len(uploader) == 2:
+            # id must be non-empty, uploader could be ''
+            uploader_id, uploader = uploader
+            uploader = uploader or None
+        else:
+            uploader_id = uploader = None
         thumbnail = sanitize_url(thumbnail)
         display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False) if type_ == 'videos' else main_id
 

From 230802bc28b927264a0858c96c18e7fa573ed860 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:33:57 +0200
Subject: [PATCH 276/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index af698d8a47b..62030e8c2d8 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -41,7 +41,7 @@ def _real_extract(self, url):
         # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1
         kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
         if not (kvs_version and kvs_version.startswith('5.')):
-            self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")
+            self.report_warning('Major version change (' + kvs_version + ') in player engine--Download may fail.')
 
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
         # video_id, video_url and license_code from the 'flashvars' JSON object:

From 9a1e32a7fcc1fde4d7962eaf0e576ac4b9673ca8 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 20:50:36 +0200
Subject: [PATCH 277/394] fix one check for Flake8

---
 youtube_dl/extractor/thisvid.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 62030e8c2d8..9f193288b9a 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -67,6 +67,8 @@ def _real_extract(self, url):
             'url': getrealurl(video_url, license_code),
             'thumbnail': thumbnail,
             'age_limit': 18,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
         }
 
 

From 64edbdd149010c9a58f4fbdebaa2a7bc12e8389e Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 14 Jun 2022 22:51:32 +0200
Subject: [PATCH 278/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 9f193288b9a..fa853153aa6 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -10,6 +10,7 @@
     sanitize_url,
 )
 
+
 class ThisVidIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'
     _TESTS = [{
@@ -31,6 +32,8 @@ class ThisVidIE(InfoExtractor):
             'title': 'Big Man Walking26',
             'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2123000/2123318/preview.mp4.jpg',
             'age_limit': 18,
+            'uploader': 'sample',
+            'uploader_id': '1064619',
         }
     }]
 

From 215d73c2c543bb7731330ef6a67f4219f3d5db1c Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 19 Jun 2022 10:54:40 +0200
Subject: [PATCH 279/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8a362a54db9..4779910168b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.06.13'
+__version__ = '2022.06.19'

From 88ab77793b18a06b434dbb9465ce01542df75190 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Jan 2022 04:28:54 +0000
Subject: [PATCH 280/394] [YouTube] Bypass age-gating for certain restricted
 videos

* Use TVHTML5_SIMPLY_EMBEDDED_PLAYER client

* Also add and fix tests

* Introduce and use new utility function `update_url()`
---
 youtube_dl/extractor/youtube.py | 202 +++++++++++++++++++++++++-------
 youtube_dl/utils.py             |  11 ++
 2 files changed, 168 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63918924df8..91de86367cf 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -40,6 +40,7 @@
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    update_url,
     update_url_query,
     url_or_none,
     urlencode_postdata,
@@ -284,15 +285,18 @@ def _real_initialize(self):
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
-    def _call_api(self, ep, query, video_id, fatal=True):
+    def _call_api(self, ep, query, video_id, fatal=True, headers=None):
         data = self._DEFAULT_API_DATA.copy()
         data.update(query)
+        real_headers = {'content-type': 'application/json'}
+        if headers:
+            real_headers.update(headers)
 
         return self._download_json(
             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
             note='Downloading API JSON', errnote='Unable to download API page',
             data=json.dumps(data).encode('utf8'), fatal=fatal,
-            headers={'content-type': 'application/json'},
+            headers=real_headers,
             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
 
     def _extract_yt_initial_data(self, video_id, webpage):
@@ -457,6 +461,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+                'channel': 'Philipp Hagemeister',
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
                 'upload_date': '20121002',
@@ -466,10 +471,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 10,
                 'view_count': int,
                 'like_count': int,
-                'dislike_count': int,
+                'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
                 'start_time': 1,
                 'end_time': 9,
-            }
+            },
         },
         {
             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
@@ -504,7 +509,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 10,
                 'view_count': int,
                 'like_count': int,
-                'dislike_count': int,
             },
             'params': {
                 'skip_download': True,
@@ -563,8 +567,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
         },
-        # Normal age-gate video (No vevo, embed allowed), available via embed page
+        # Age-gated videos
         {
+            'note': 'Age-gated video (No vevo, embed allowed)',
             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
             'info_dict': {
                 'id': 'HtVdAasjOgU',
@@ -573,17 +578,97 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'duration': 142,
                 'uploader': 'The Witcher',
-                'uploader_id': 'WitcherGame',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
+                'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
+                'age_limit': 18,
+                'categories': ['Gaming'],
+                'tags': 'count:17',
+                'channel': 'The Witcher',
+                'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
+                'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Age-gated video with embed allowed in public site',
+            'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
+            'info_dict': {
+                'id': 'HsUATh_Nc2U',
+                'ext': 'mp4',
+                'title': 'Godzilla 2 (Official Video)',
+                'description': 'md5:bf77e03fcae5529475e500129b05668a',
+                'duration': 177,
+                'uploader': 'FlyingKitty',
+                'upload_date': '20200408',
+                'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
                 'age_limit': 18,
+                'categories': ['Entertainment'],
+                'tags': ['Flyingkitty', 'godzilla 2'],
+                'channel': 'FlyingKitty',
+                'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
+                'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
+                'view_count': int,
+                'like_count': int,
             },
         },
         {
-            # Age-gated video only available with authentication (unavailable
-            # via embed page workaround)
+            'note': 'Age-gated video embedable only with clientScreen=EMBED',
+            'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
+            'info_dict': {
+                'id': 'Tq92D6wQ1mg',
+                'ext': 'mp4',
+                'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
+                'description': 'md5:17eccca93a786d51bc67646756894066',
+                'duration': 106,
+                'uploader': 'Projekt Melody',
+                'upload_date': '20191227',
+                'age_limit': 18,
+                'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
+                'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
+                'categories': ['Entertainment'],
+                'channel': 'Projekt Melody',
+                'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Non-Age-gated non-embeddable video',
+            'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
+            'info_dict': {
+                'id': 'MeJVWBSsPAY',
+                'ext': 'mp4',
+                'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
+                'description': 'Fan Video. Music & Lyrics by OOMPH!.',
+                'duration': 210,
+                'uploader': 'Herr Lurik',
+                'uploader_id': 'st3in234',
+                'upload_date': '20130730',
+                'uploader_url': 'http://www.youtube.com/user/st3in234',
+                'age_limit': 0,
+                'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/hqdefault.jpg',
+                'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
+                'categories': ['Music'],
+                'channel': 'Herr Lurik',
+                'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
+                'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
+                'artist': 'OOMPH!',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Non-bypassable age-gated video',
+            'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
+            'only_matching': True,
+        },
+        {
+            'note': 'Age-gated video only available with authentication (not via embed workaround)',
             'url': 'XgnwCQzjau8',
             'only_matching': True,
+            'skip': '''This video has been removed for violating YouTube's Community Guidelines''',
         },
         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
         # YouTube Red ad is not captured for creator
@@ -612,17 +697,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
+                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+                'description': r're:(?s)(?:.+\s)?HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games\s*',
                 'duration': 6085,
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
-                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
-                'uploader': 'Olympic',
-                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+                'uploader': r're:Olympics?',
+                'age_limit': 0,
+                'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
+                'categories': ['Sports'],
+                'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
+                'channel': 'Olympics',
+                'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
+                'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
+                'view_count': int,
+                'like_count': int,
             },
-            'params': {
-                'skip_download': 'requires avconv',
-            }
         },
         # Non-square pixels
         {
@@ -1586,27 +1677,52 @@ def _real_extract(self, url):
             player_response = self._call_api(
                 'player', {'videoId': video_id}, video_id)
 
-        playability_status = player_response.get('playabilityStatus') or {}
-        if playability_status.get('reason') == 'Sign in to confirm your age':
-            video_info = self._download_webpage(
-                base_url + 'get_video_info', video_id,
-                'Refetching age-gated info webpage',
-                'unable to download video info webpage', query={
-                    'video_id': video_id,
-                    'eurl': 'https://youtube.googleapis.com/v/' + video_id,
-                    'html5': 1,
-                    # See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
-                    'c': 'TVHTML5',
-                    'cver': '6.20180913',
-                }, fatal=False)
-            if video_info:
-                pr = self._parse_json(
-                    try_get(
-                        compat_parse_qs(video_info),
-                        lambda x: x['player_response'][0], compat_str) or '{}',
-                    video_id, fatal=False)
-                if pr and isinstance(pr, dict):
-                    player_response = pr
+        def is_agegated(playability):
+            if not isinstance(playability, dict):
+                return
+
+            if playability.get('desktopLegacyAgeGateReason'):
+                return True
+
+            reasons = filter(None, (playability.get(r) for r in ('status', 'reason')))
+            AGE_GATE_REASONS = (
+                'confirm your age', 'age-restricted', 'inappropriate',  # reason
+                'age_verification_required', 'age_check_required',  # status
+            )
+            return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
+
+        def get_playability_status(response):
+            return try_get(response, lambda x: x['playabilityStatus'], dict) or {}
+
+        playability_status = get_playability_status(player_response)
+        if (is_agegated(playability_status)
+                and int_or_none(self._downloader.params.get('age_limit'), default=18) >= 18):
+
+            self.report_age_confirmation()
+
+            # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
+            pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
+            query = {
+                'playbackContext': {'contentPlaybackContext': {'html5Preference': 'HTML5_PREF_WANTS'}},
+                'contentCheckOk': True,
+                'racyCheckOk': True,
+                'context': {
+                    'client': {'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'clientVersion': '2.0', 'hl': 'en', 'clientScreen': 'EMBED'},
+                    'thirdParty': {'embedUrl': 'https://google.com'},
+                },
+                'videoId': video_id,
+            }
+            headers = {
+                'X-YouTube-Client-Name': '85',
+                'X-YouTube-Client-Version': '2.0',
+                'Origin': 'https://www.youtube.com'
+            }
+
+            video_info = self._call_api('player', query, video_id, fatal=False, headers=headers)
+            age_gate_status = get_playability_status(video_info)
+            if age_gate_status.get('status') == 'OK':
+                player_response = video_info
+                playability_status = age_gate_status
 
         trailer_video_id = try_get(
             playability_status,
@@ -1835,12 +1951,12 @@ def feed_entry(name):
             for thumbnail in (try_get(
                     container,
                     lambda x: x['thumbnail']['thumbnails'], list) or []):
-                thumbnail_url = thumbnail.get('url')
+                thumbnail_url = url_or_none(thumbnail.get('url'))
                 if not thumbnail_url:
                     continue
                 thumbnails.append({
                     'height': int_or_none(thumbnail.get('height')),
-                    'url': thumbnail_url,
+                    'url': update_url(thumbnail_url, query=None, fragment=None),
                     'width': int_or_none(thumbnail.get('width')),
                 })
             if thumbnails:
@@ -2045,6 +2161,7 @@ def chapter_time(mmlir):
                     sbr_tooltip = try_get(
                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
                     if sbr_tooltip:
+                        # however dislike_count was hidden by YT, as if there could ever be dislikable content on YT
                         like_count, dislike_count = sbr_tooltip.split(' / ')
                         info.update({
                             'like_count': str_to_int(like_count),
@@ -2296,7 +2413,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'tags': list,
             'view_count': int,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -2323,7 +2439,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'categories': ['News & Politics'],
             'tags': list,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -2343,7 +2458,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'categories': ['News & Politics'],
             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -2910,8 +3024,7 @@ def _extract_identity_token(self, ytcfg, webpage):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        url = compat_urlparse.urlunparse(
-            compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
+        url = update_url(url, netloc='www.youtube.com')
         # Handle both video/playlist URLs
         qs = parse_qs(url)
         video_id = qs.get('v', [None])[0]
@@ -3045,7 +3158,6 @@ class YoutubeYtBeIE(InfoExtractor):
             'categories': ['Nonprofits & Activism'],
             'tags': list,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'noplaylist': True,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58de..4f93bdde739 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3957,6 +3957,17 @@ def update_url_query(url, query):
         query=compat_urllib_parse_urlencode(qs, True)))
 
 
+def update_url(url, **kwargs):
+    """Replace URL components specified by kwargs
+       url: compat_str or parsed URL tuple
+       returns: compat_str"""
+    if not kwargs:
+        return compat_urlparse.urlunparse(url) if isinstance(url, tuple) else url
+    if not isinstance(url, tuple):
+        url = compat_urlparse.urlparse(url)
+    return compat_urlparse.urlunparse(url._replace(**kwargs))
+
+
 def update_Request(req, url=None, data=None, headers={}, query={}):
     req_headers = req.headers.copy()
     req_headers.update(headers)

From 7e1b181df4f07e78853bb57ef5c81afa599c2a16 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 20 Jun 2022 23:15:20 +0100
Subject: [PATCH 281/394] [YouTube] Add `signatureTimestamp` for age-gate
 bypass

---
 youtube_dl/extractor/youtube.py | 34 +++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 91de86367cf..749a84950f2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1545,6 +1545,27 @@ def _unthrottle_format_urls(self, video_id, player_url, formats):
                 fmt['url'] = compat_urlparse.urlunparse(
                     parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
+    # from yt-dlp, with tweaks
+    def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
+        """
+        Extract signatureTimestamp (sts)
+        Required to tell API what sig/player version is in use.
+        """
+        sts = int_or_none(ytcfg.get('STS')) if isinstance(ytcfg, dict) else None
+        if not sts:
+            # Attempt to extract from player
+            if player_url is None:
+                error_msg = 'Cannot extract signature timestamp without player_url.'
+                if fatal:
+                    raise ExtractorError(error_msg)
+                self._downloader.report_warning(error_msg)
+                return
+            code = self._get_player_code(video_id, player_url)
+            sts = int_or_none(self._search_regex(
+                r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
+                'JS player signature timestamp', group='sts', fatal=fatal))
+        return sts
+
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
@@ -1669,6 +1690,7 @@ def _real_extract(self, url):
             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
 
         player_response = None
+        player_url = None
         if webpage:
             player_response = self._extract_yt_initial_variable(
                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
@@ -1702,8 +1724,17 @@ def get_playability_status(response):
 
             # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
+
+            # Use signatureTimestamp if available
+            # Thanks https://github.com/ytdl-org/youtube-dl/issues/31034#issuecomment-1160718026
+            player_url = self._extract_player_url(webpage)
+            ytcfg = self._extract_ytcfg(video_id, webpage)
+            sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
+            if sts:
+                pb_context['signatureTimestamp'] = sts
+
             query = {
-                'playbackContext': {'contentPlaybackContext': {'html5Preference': 'HTML5_PREF_WANTS'}},
+                'playbackContext': {'contentPlaybackContext': pb_context},
                 'contentCheckOk': True,
                 'racyCheckOk': True,
                 'context': {
@@ -1804,7 +1835,6 @@ def feed_entry(name):
         formats = []
         itags = []
         itag_qualities = {}
-        player_url = None
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
         streaming_data = player_response.get('streamingData') or {}
         streaming_formats = streaming_data.get('formats') or []

From d9798f8facb2973d7b7f3bda36bc057d628e5a7f Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 19:59:30 +0200
Subject: [PATCH 282/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index fa853153aa6..8be73ef6d34 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (

From ded4672eb7b888ff72ca73b40f6f22362938479b Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 19:59:38 +0200
Subject: [PATCH 283/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 8be73ef6d34..e6d974c781c 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -9,7 +9,11 @@
     compat_str,
 )
 from ..utils import (
+    clean_html,
+    get_element_by_class,
     sanitize_url,
+    url_or_none,
+    urljoin,
 )
 
 

From 795bd30a7b5a41c46a60324d4d741602dc40405f Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 19:59:50 +0200
Subject: [PATCH 284/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index e6d974c781c..cb4ad545bd8 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -20,26 +20,28 @@
 class ThisVidIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'
     _TESTS = [{
-        'url': 'https://thisvid.com/videos/big-man-walking26/',
-        'md5': '74a11cac6a738f28c1a37945145a3f96',
+        'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
+        'md5': '839becb572995687e11a69dc4358a386',
         'info_dict': {
-            'id': '2123318',
+            'id': '3533241',
             'ext': 'mp4',
-            'title': 'Big Man Walking26',
-            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2123000/2123318/preview.mp4.jpg',
+            'title': 'Sitting on ball tight jeans',
+            'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
+            'uploader_id': '150629',
+            'uploader': 'jeanslevisjeans',
             'age_limit': 18,
         }
     }, {
-        'url': 'https://thisvid.com/embed/2123318/',
-        'md5': '74a11cac6a738f28c1a37945145a3f96',
+        'url': 'https://thisvid.com/embed/3533241/',
+        'md5': '839becb572995687e11a69dc4358a386',
         'info_dict': {
-            'id': '2123318',
+            'id': '3533241',
             'ext': 'mp4',
-            'title': 'Big Man Walking26',
-            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2123000/2123318/preview.mp4.jpg',
+            'title': 'Sitting on ball tight jeans',
+            'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
+            'uploader_id': '150629',
+            'uploader': 'jeanslevisjeans',
             'age_limit': 18,
-            'uploader': 'sample',
-            'uploader_id': '1064619',
         }
     }]
 

From 93289fec98bac4f2beb1302399c16c264466dde4 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:01 +0200
Subject: [PATCH 285/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index cb4ad545bd8..48f872de1f0 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -55,7 +55,11 @@ def _real_extract(self, url):
             self.report_warning('Major version change (' + kvs_version + ') in player engine--Download may fail.')
 
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
-        # video_id, video_url and license_code from the 'flashvars' JSON object:
+        video_holder = get_element_by_class('video-holder', webpage) or ''
+       if '>This video is a private video' in video_holder:
+           self.raise_login_required(
+               (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
+       # video_id, video_url and license_code from the 'flashvars' JSON object:
         video_id = self._html_search_regex(r"video_id:\s+'([0-9]+)',", webpage, 'video_id')
         video_url = self._html_search_regex(r"video_url:\s+'function/0/((?:[^/']+/){5,}.+?)',", webpage, 'video_url')
         license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')

From cf2d90376a08181bbcbac1b773bb4a2b74b853e9 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:21 +0200
Subject: [PATCH 286/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 48f872de1f0..d066355fb0b 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -77,7 +77,7 @@ def _real_extract(self, url):
 
         return {
             'id': video_id,
-            'display_id': display_id,
+            'display_id': main_id,
             'title': title,
             'url': getrealurl(video_url, license_code),
             'thumbnail': thumbnail,

From f692a9cc5386d3aa94adbc16f531cc003cb240c9 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:32 +0200
Subject: [PATCH 287/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d066355fb0b..41efdef9205 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -88,7 +88,7 @@ def _real_extract(self, url):
 
 
 def getrealurl(video_url, license_code):
-    urlparts = video_url.split('/')[2:]
+    urlparts = video_url.split('/')
     license = getlicensetoken(license_code)
     newmagic = urlparts[5][:32]
 

From 7c64270fded132719e56ea2d79fb68361149ec2b Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:39 +0200
Subject: [PATCH 288/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 41efdef9205..94a47c0bd1e 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -106,7 +106,7 @@ def getrealurl(video_url, license_code):
         newmagic = new
 
     urlparts[5] = newmagic + urlparts[5][32:]
-    return "/".join(urlparts)
+    return '/'.join(urlparts)
 
 
 def getlicensetoken(license):

From e5697c9afbff38e2707a977076fad24d2d910905 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:45 +0200
Subject: [PATCH 289/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 94a47c0bd1e..998e5397274 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -110,7 +110,7 @@ def getrealurl(video_url, license_code):
 
 
 def getlicensetoken(license):
-    modlicense = license.replace("$", "").replace("0", "1")
+    modlicense = license.replace('$', '').replace('0', '1')
     center = len(modlicense) // 2
     fronthalf = int(modlicense[:center + 1])
     backhalf = int(modlicense[center:])

From 4c58266d9d9573da7392b3fdea58053091ba6ca8 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:00:54 +0200
Subject: [PATCH 290/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 55 +++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 998e5397274..af9bc8116c2 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -121,3 +121,58 @@ def getlicensetoken(license):
         for i in range(1, 5):
             retval += compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
     return retval
+
+
+class ThisVidMemberIE(InfoExtractor):
+    _VALID_URL = r'https?://thisvid\.com/members/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/members/3235959/',
+        'info_dict': {
+            'id': '3235959',
+            'title': 'ButtLoverAss\'s Profile',
+        },
+        'playlist_mincount': 64,
+    }, {
+        'url': 'https://thisvid.com/members/3235959/favourite_videos/',
+        'info_dict': {
+            'id': '3235959',
+            'title': 'ButtLoverAss\'s Favourite Videos',
+        },
+        'playlist_mincount': 57,
+    }, {
+        'url': 'https://thisvid.com/members/3235959/public_videos/',
+        'info_dict': {
+            'id': '3235959',
+            'title': 'ButtLoverAss\'s Public Videos',
+        },
+        'playlist_mincount': 29,
+    },
+    ]
+
+    def _real_extract(self, url):
+        pl_id = self._match_id(url)
+        webpage = self._download_webpage(url, pl_id)
+
+        title = re.split(
+            r'(?i)\s*\|\s*ThisVid\.com\s*$',
+            self._og_search_title(webpage, default=None) or self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title', webpage, 'title', fatal=False) or '', 1)[0] or None
+
+        def entries(page_url, html=None):
+            for page in itertools.count(1):
+                if not html:
+                    html = self._download_webpage(
+                        page_url, pl_id, note='Downloading page %d' % (page, ),
+                        fatal=False) or ''
+                for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
+                    yield m.group('url')
+                next_page = get_element_by_class('pagination-next', html)
+                next_page = urljoin(url,
+                    next_page and self._search_regex(
+                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''', 
+                        next_page, 'next page link', group='url', default=None))
+                if not next_page:
+                    break
+                page_url, html = next_page, None
+
+        return self.playlist_from_matches(
+            entries(url, webpage), playlist_id=pl_id, playlist_title=title, ie='ThisVid')

From ecddec2208f0c95faaf7f5a204212baa54929954 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:01:06 +0200
Subject: [PATCH 291/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index af9bc8116c2..f27f440e7c8 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -55,6 +55,10 @@ def _real_extract(self, url):
             self.report_warning('Major version change (' + kvs_version + ') in player engine--Download may fail.')
 
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
+        if type_ == 'embed':
+            video_alt_url = url_or_none(self._html_search_regex(r'''video_alt_url:\s+'(%s)',''' % (self._VALID_URL, ), webpage, 'video_alt_url', default=None))
+            if video_alt_url:
+                webpage = self._download_webpage(video_alt_url, main_id, note='Redirecting embed to main page', fatal=False)
         video_holder = get_element_by_class('video-holder', webpage) or ''
        if '>This video is a private video' in video_holder:
            self.raise_login_required(

From b26d15b049cecb2d669d5d7c7ed0b29fe05cc074 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:01:16 +0200
Subject: [PATCH 292/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index f27f440e7c8..97784137929 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -64,8 +64,8 @@ def _real_extract(self, url):
            self.raise_login_required(
                (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
        # video_id, video_url and license_code from the 'flashvars' JSON object:
-        video_id = self._html_search_regex(r"video_id:\s+'([0-9]+)',", webpage, 'video_id')
-        video_url = self._html_search_regex(r"video_url:\s+'function/0/((?:[^/']+/){5,}.+?)',", webpage, 'video_url')
+        video_id = self._html_search_regex(r'''video_id:\s+'([0-9]+)',''', webpage, 'video_id')
+        video_url = self._html_search_regex(r'''video_url:\s+'function/0/(https?://(?:[^/']+/){5,}.*?)',''', webpage, 'video_url')
         license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')
         thumbnail = self._html_search_regex(r"preview_url:\s+'((?:https?:)?//media\.thisvid\.com/.+?\.jpg)',", webpage, 'thumbnail', fatal=False)
         uploader = self._html_search_regex(r'''(?s)<span\b[^>]*>Added by:\s*</span><a\b[^>]+\bclass\s*=\s*["']author\b[^>]+\bhref\s*=\s*["']https://thisvid\.com/members/([0-9]+/.{3,}?)\s*</a>''', webpage, 'uploader', default='')

From 361c036e10b98a8e59aacdb3ec87dea9aa3372c2 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:01:30 +0200
Subject: [PATCH 293/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 97784137929..d19d60a83d9 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -77,7 +77,6 @@ def _real_extract(self, url):
         else:
             uploader_id = uploader = None
         thumbnail = sanitize_url(thumbnail)
-        display_id = self._search_regex(r'<link\s+rel\s*=\s*"canonical"\s+href\s*=\s*"%s"' % (self._VALID_URL, ), webpage, 'display_id', fatal=False) if type_ == 'videos' else main_id
 
         return {
             'id': video_id,

From 9cd76aef2c74e1c27c84f844b4a677f9b6efb5c5 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Thu, 23 Jun 2022 20:05:39 +0200
Subject: [PATCH 294/394] Update extractors.py

added ThisVidMemberIE
---
 youtube_dl/extractor/extractors.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a61a8716f8f..0aaa9d3b03c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1264,7 +1264,10 @@
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
-from .thisvid import ThisVidIE
+from .thisvid import (
+    ThisVidIE,
+    ThisVidMemberIE,
+)
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,

From 8060d5c317b6f02228223f655ae65b688179bfc9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 23 Jun 2022 23:25:30 +0100
Subject: [PATCH 295/394] Update youtube_dl/extractor/thisvid.py

---
 youtube_dl/extractor/thisvid.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d19d60a83d9..11452768791 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -60,10 +60,10 @@ def _real_extract(self, url):
             if video_alt_url:
                 webpage = self._download_webpage(video_alt_url, main_id, note='Redirecting embed to main page', fatal=False)
         video_holder = get_element_by_class('video-holder', webpage) or ''
-       if '>This video is a private video' in video_holder:
-           self.raise_login_required(
-               (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
-       # video_id, video_url and license_code from the 'flashvars' JSON object:
+        if '>This video is a private video' in video_holder:
+            self.raise_login_required(
+                (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
+        # video_id, video_url and license_code from the 'flashvars' JSON object:
         video_id = self._html_search_regex(r'''video_id:\s+'([0-9]+)',''', webpage, 'video_id')
         video_url = self._html_search_regex(r'''video_url:\s+'function/0/(https?://(?:[^/']+/){5,}.*?)',''', webpage, 'video_url')
         license_code = self._html_search_regex(r"license_code:\s+'([0-9$]{16})',", webpage, 'license_code')

From e4f130b3d24ce4e335847bfb35f3a2ff4b031d0f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 23 Jun 2022 23:29:40 +0100
Subject: [PATCH 296/394] Update youtube_dl/extractor/thisvid.py

---
 youtube_dl/extractor/thisvid.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 11452768791..1f893e5d49c 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,9 +169,9 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url,
-                    next_page and self._search_regex(
-                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''', 
+                next_page = urljoin(url, next_page and 
+                    self._search_regex(
+                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
                         next_page, 'next page link', group='url', default=None))
                 if not next_page:
                     break

From 312c1fdc58df1d3d7be166385cfda5e813b7e062 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 23 Jun 2022 23:30:21 +0100
Subject: [PATCH 297/394] Update youtube_dl/extractor/thisvid.py

---
 youtube_dl/extractor/thisvid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 1f893e5d49c..f295248468d 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,8 +169,8 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url, next_page and 
-                    self._search_regex(
+                next_page = urljoin(url, next_page
+                     and self._search_regex(
                         r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
                         next_page, 'next page link', group='url', default=None))
                 if not next_page:

From 4c42bbaac9be47bcb7d292546becdde79f8b3fb5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 24 Jun 2022 00:05:27 +0100
Subject: [PATCH 298/394] Update youtube_dl/extractor/thisvid.py

---
 youtube_dl/extractor/thisvid.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index f295248468d..2757c030679 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,10 +169,11 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url, next_page
-                     and self._search_regex(
-                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                        next_page, 'next page link', group='url', default=None))
+                next_page = urljoin(url,
+                    next_page
+                    and self._search_regex(
+                            r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
+                            next_page, 'next page link', group='url', default=None))
                 if not next_page:
                     break
                 page_url, html = next_page, None

From 5411b9b6685a1f25841f8caf674c68fa569c2c9b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 24 Jun 2022 00:25:06 +0100
Subject: [PATCH 299/394] Update youtube_dl/extractor/thisvid.py

---
 youtube_dl/extractor/thisvid.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 2757c030679..50cb89686c3 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,11 +169,11 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url,
-                    next_page
-                    and self._search_regex(
-                            r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                            next_page, 'next page link', group='url', default=None))
+                next_page = urljoin(
+                    url, next_page
+                         and self._search_regex(
+                             r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
+                             next_page, 'next page link', group='url', default=None))
                 if not next_page:
                     break
                 page_url, html = next_page, None

From a097485056cab50f1b79ef03d8e171b01ed6bc3a Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Fri, 24 Jun 2022 09:18:39 +0200
Subject: [PATCH 300/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 50cb89686c3..b1c612f5757 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,11 +169,10 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(
-                    url, next_page
-                         and self._search_regex(
-                             r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                             next_page, 'next page link', group='url', default=None))
+                next_page = urljoin(url, 
+                    self._search_regex(
+                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
+                        next_page, 'next page link', group='url', default=None) if next_page else None)
                 if not next_page:
                     break
                 page_url, html = next_page, None

From 1295c593e101b4977028ee161369d30831d090cc Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Fri, 24 Jun 2022 17:34:01 +0200
Subject: [PATCH 301/394] update indentation

---
 youtube_dl/extractor/thisvid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index b1c612f5757..1d8d902818e 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,8 +169,8 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url, 
-                    self._search_regex(
+                next_page = urljoin(url,
+                        self._search_regex(
                         r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
                         next_page, 'next page link', group='url', default=None) if next_page else None)
                 if not next_page:

From 44d724a9e301e2ee4dc7c937366b1b9049eed5e9 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Fri, 24 Jun 2022 17:52:10 +0200
Subject: [PATCH 302/394] Update thisvid.py

hopefully last indentation update
---
 youtube_dl/extractor/thisvid.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 1d8d902818e..d6b40cd37dd 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -169,8 +169,7 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url,
-                        self._search_regex(
+                next_page = urljoin(url, self._search_regex(
                         r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
                         next_page, 'next page link', group='url', default=None) if next_page else None)
                 if not next_page:

From 8318f7dde5bb227fef8aa93f890f2244bc425a4d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 24 Jun 2022 17:33:03 +0100
Subject: [PATCH 303/394] Update youtube_dl/extractor/thisvid.py

again
---
 youtube_dl/extractor/thisvid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d6b40cd37dd..85fccbcc6d0 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -170,8 +170,8 @@ def entries(page_url, html=None):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
                 next_page = urljoin(url, self._search_regex(
-                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                        next_page, 'next page link', group='url', default=None) if next_page else None)
+                    r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
+                    next_page, 'next page link', group='url', default=None) if next_page else None)
                 if not next_page:
                     break
                 page_url, html = next_page, None

From a4be775148df62bade861fd4aa169fac0d89c6f2 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sun, 26 Jun 2022 09:00:58 +0200
Subject: [PATCH 304/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 85fccbcc6d0..298d9cff302 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -49,11 +49,6 @@ def _real_extract(self, url):
         main_id, type_ = re.match(self._VALID_URL, url).group('id', 'type')
         webpage = self._download_webpage(url, main_id)
 
-        # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1
-        kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
-        if not (kvs_version and kvs_version.startswith('5.')):
-            self.report_warning('Major version change (' + kvs_version + ') in player engine--Download may fail.')
-
         title = self._html_search_regex(r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>', webpage, 'title')
         if type_ == 'embed':
             video_alt_url = url_or_none(self._html_search_regex(r'''video_alt_url:\s+'(%s)',''' % (self._VALID_URL, ), webpage, 'video_alt_url', default=None))
@@ -63,6 +58,12 @@ def _real_extract(self, url):
         if '>This video is a private video' in video_holder:
             self.raise_login_required(
                 (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
+
+        # URL decryptor was reversed from version 4.0.4, later verified working with 5.0.1
+        kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
+        if not (kvs_version and kvs_version.startswith('5.')):
+            self.report_warning('Major version change (' + kvs_version + ') in player engine--Download may fail.')
+
         # video_id, video_url and license_code from the 'flashvars' JSON object:
         video_id = self._html_search_regex(r'''video_id:\s+'([0-9]+)',''', webpage, 'video_id')
         video_url = self._html_search_regex(r'''video_url:\s+'function/0/(https?://(?:[^/']+/){5,}.*?)',''', webpage, 'video_url')

From 51cf04ada81362b2c5f463de409f77931c31b5e6 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sun, 26 Jun 2022 09:11:24 +0200
Subject: [PATCH 305/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 72 ++++++++++++++++-----------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 298d9cff302..d1b457db8c1 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -79,6 +79,42 @@ def _real_extract(self, url):
             uploader_id = uploader = None
         thumbnail = sanitize_url(thumbnail)
 
+
+        def getrealurl(video_url, license_code):
+            urlparts = video_url.split('/')
+            license = getlicensetoken(license_code)
+            newmagic = urlparts[5][:32]
+
+            for o in range(len(newmagic) - 1, -1, -1):
+                new = ''
+                l = (o + sum([int(n) for n in license[o:]])) % 32
+
+                for i in range(0, len(newmagic)):
+                    idx = i
+                    if idx == o:
+                        idx = l
+                    elif idx == l:
+                        idx = o
+                    new += newmagic[idx]
+                newmagic = new
+
+            urlparts[5] = newmagic + urlparts[5][32:]
+            return '/'.join(urlparts)
+
+
+        def getlicensetoken(license):
+            modlicense = license.replace('$', '').replace('0', '1')
+            center = len(modlicense) // 2
+            fronthalf = int(modlicense[:center + 1])
+            backhalf = int(modlicense[center:])
+
+            modlicense = compat_str(4 * abs(fronthalf - backhalf))
+            retval = ''
+            for o in range(0, center + 1):
+                for i in range(1, 5):
+                    retval += compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
+            return retval
+
         return {
             'id': video_id,
             'display_id': main_id,
@@ -91,42 +127,6 @@ def _real_extract(self, url):
         }
 
 
-def getrealurl(video_url, license_code):
-    urlparts = video_url.split('/')
-    license = getlicensetoken(license_code)
-    newmagic = urlparts[5][:32]
-
-    for o in range(len(newmagic) - 1, -1, -1):
-        new = ''
-        l = (o + sum([int(n) for n in license[o:]])) % 32
-
-        for i in range(0, len(newmagic)):
-            idx = i
-            if idx == o:
-                idx = l
-            elif idx == l:
-                idx = o
-            new += newmagic[idx]
-        newmagic = new
-
-    urlparts[5] = newmagic + urlparts[5][32:]
-    return '/'.join(urlparts)
-
-
-def getlicensetoken(license):
-    modlicense = license.replace('$', '').replace('0', '1')
-    center = len(modlicense) // 2
-    fronthalf = int(modlicense[:center + 1])
-    backhalf = int(modlicense[center:])
-
-    modlicense = compat_str(4 * abs(fronthalf - backhalf))
-    retval = ''
-    for o in range(0, center + 1):
-        for i in range(1, 5):
-            retval += compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
-    return retval
-
-
 class ThisVidMemberIE(InfoExtractor):
     _VALID_URL = r'https?://thisvid\.com/members/(?P<id>\d+)'
     _TESTS = [{

From 01de88a2418fcbbc9fbb2ceae50fc24ff6a1b0c9 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sun, 26 Jun 2022 17:38:56 +0200
Subject: [PATCH 306/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index d1b457db8c1..e5fda6d89d2 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -79,7 +79,6 @@ def _real_extract(self, url):
             uploader_id = uploader = None
         thumbnail = sanitize_url(thumbnail)
 
-
         def getrealurl(video_url, license_code):
             urlparts = video_url.split('/')
             license = getlicensetoken(license_code)
@@ -101,7 +100,6 @@ def getrealurl(video_url, license_code):
             urlparts[5] = newmagic + urlparts[5][32:]
             return '/'.join(urlparts)
 
-
         def getlicensetoken(license):
             modlicense = license.replace('$', '').replace('0', '1')
             center = len(modlicense) // 2

From 951aa74cbeb08a4220e265e769a263dcedf46282 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Mon, 27 Jun 2022 13:58:07 +0200
Subject: [PATCH 307/394] [livestreamfails] Add new extractor

---
 youtube_dl/extractor/extractors.py      |  3 ++
 youtube_dl/extractor/livestreamfails.py | 49 +++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/livestreamfails.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..c01461f5476 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -627,6 +627,9 @@
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
+from .livestreamfails import (
+    LivestreamfailsIE,
+)
 from .lnkgo import LnkGoIE
 from .localnews8 import LocalNews8IE
 from .lovehomeporn import LoveHomePornIE
diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
new file mode 100644
index 00000000000..b92eca04700
--- /dev/null
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -0,0 +1,49 @@
+from .common import InfoExtractor
+import json
+import time
+import calendar
+
+
+class LivestreamfailsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?livestreamfails\.com/clip/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://livestreamfails.com/clip/139200',
+        'md5': '8a03aea1a46e94a05af6410337463102',
+        'info_dict': {
+            'id': '139200',
+            'ext': 'mp4',
+            'display_id': 'ConcernedLitigiousSalmonPeteZaroll-O8yo9W2L8OZEKhV2',
+            'title': 'Streamer jumps off a trampoline at full speed',
+            'creator': 'paradeev1ch',
+            'thumbnail': 'https://livestreamfails-image-prod.b-cdn.net/image/3877b1d38db083fa25c82685bbaf645637e575ea.png',
+            'timestamp': 1656271785,
+            'upload_date': '20220626',
+        }
+    }]
+
+    def _real_extract(self, url):
+        result = {}
+        result['id'] = self._match_id(url)
+
+        # https://livestreamfails.com/clip/id uses https://api.livestreamfails.com/clip/ to fetch the video metadata
+        # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
+        apiResponse = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + result['id'], result['id']))
+
+        # Twitch ID of clip
+        result['display_id'] = apiResponse.get('sourceId')
+
+        # Get the input timestamp (test case gives 2022-06-26T19:29:45.515Z)
+        result['timestamp'] = apiResponse.get('createdAt')
+        if(result.get('timestamp')):
+            # Parse it into a struct_time
+            result['timestamp'] = time.strptime(result['timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            # Convert the struct_time to a UNIX timestamp while ignoring the local timezone attached by time.strptime()
+            result['timestamp'] = calendar.timegm(result['timestamp'])
+
+        # Other fields
+        result['url'] = 'https://livestreamfails-video-prod.b-cdn.net/video/' + apiResponse.get('videoId')
+        result['title'] = apiResponse.get('label')
+        result['creator'] = apiResponse.get('streamer', {}).get('label')
+        result['thumbnail'] = 'https://livestreamfails-image-prod.b-cdn.net/image/' + apiResponse.get('imageId')
+
+        return result

From 295c8ca6bd8d43bae999edd669d71d0ddd90151c Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Thu, 30 Jun 2022 19:36:50 +0200
Subject: [PATCH 308/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4779910168b..6bf47e50497 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.06.19'
+__version__ = '2022.06.30'

From 175bdaf09027bed8ef4173e36694d9318b60bf46 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Mon, 4 Jul 2022 16:42:56 +0200
Subject: [PATCH 309/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index e5fda6d89d2..37c85c306f1 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -18,7 +18,7 @@
 
 
 class ThisVidIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'
+    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed|(?P<pl>playlist))(?(pl)/\d+/video)/(?P<id>[A-Za-z0-9-]+/?)'
     _TESTS = [{
         'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
         'md5': '839becb572995687e11a69dc4358a386',

From 58d9be8fb8d2e097f6827a21f9b83a415940d385 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Mon, 4 Jul 2022 18:32:47 +0200
Subject: [PATCH 310/394] Update youtube_dl/extractor/extractors.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0aaa9d3b03c..c6aae6e3a48 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1267,6 +1267,7 @@
 from .thisvid import (
     ThisVidIE,
     ThisVidMemberIE,
+    ThisVidPlaylistIE,
 )
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (

From 7e028f23cf0cea39d3e66279c2d3df59e15a2c58 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Mon, 4 Jul 2022 18:33:03 +0200
Subject: [PATCH 311/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 37c85c306f1..96bb132c7dd 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -18,7 +18,7 @@
 
 
 class ThisVidIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed|(?P<pl>playlist))(?(pl)/\d+/video)/(?P<id>[A-Za-z0-9-]+/?)'
+    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
         'md5': '839becb572995687e11a69dc4358a386',

From e599b787e3eb8991908254fcc8d9ff32cf34e96d Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Mon, 4 Jul 2022 18:33:29 +0200
Subject: [PATCH 312/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 96bb132c7dd..a3166e6e003 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -168,9 +168,23 @@ def entries(page_url, html=None):
                 for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
                     yield m.group('url')
                 next_page = get_element_by_class('pagination-next', html)
-                next_page = urljoin(url, self._search_regex(
-                    r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                    next_page, 'next page link', group='url', default=None) if next_page else None)
+                if next_page:
+                    # member list page
+                    next_page = urljoin(url, self._search_regex(
+                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
+                        next_page, 'next page link', group='url', default=None)
+                # in case a member page should have pagination-next with empty link, not just `else:`
+                if next_page is None:
+                    # playlist page
+                    parsed_url = compat_urlparse.urlparse(page_url)
+                    base_path, num = parsed_url.path.rsplit('/', 1)
+                    num = int_or_none(num)
+                    if num is None:
+                        base_path, num = parsed_url.path.rstrip('/'), 1
+                    parsed_url._replace(path=base_path + ('/%d' % (num + 1, )))
+                    next_page = compat_urlparse.urlunparse(parsed_url)
+                    if page_url == next_page:
+                        next_page = None
                 if not next_page:
                     break
                 page_url, html = next_page, None

From a938aedc4366094637cdbc18fd8fd8f7d427f9ae Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Mon, 4 Jul 2022 18:33:46 +0200
Subject: [PATCH 313/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 42 +++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index a3166e6e003..b97cd48e827 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -191,3 +191,45 @@ def entries(page_url, html=None):
 
         return self.playlist_from_matches(
             entries(url, webpage), playlist_id=pl_id, playlist_title=title, ie='ThisVid')
+
+
+class ThisVidPlaylistIE(ThisVidMemberIE):
+    _VALID_URL = r'https?://thisvid\.com/playlist/(?P<id>\d+)/video/(?P<video_id>[A-Za-z0-9-]+)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
+        'info_dict': {
+            'id': '6615',
+            'title': 'Underwear Stuff',
+        },
+        'playlist_mincount': 207,
+    }, {
+        'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
+        'info_dict': {
+            'id': '1072387',
+            'ext': 'mp4',
+            'title': 'Big Italian Booty 28',
+        'params': {
+            'noplaylist: True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        pl_id, video_id = re.match(self._VALID_URL, url).groups('id', 'video_id')
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen('Downloading just the featured video because of --no-playlist')
+            return self.url_result(urljoin(url, '/videos/' + video_id), 'ThisVid', video_id)
+
+        self.to_screen(
+            'Downloading playlist {pl_id} - add --no-playlist to download just the featured video').format(locals())
+        
+        result = super(ThisVidPlaylistIE, self)._real_extract(url)
+
+        # rework title returned as `the title - the title`
+        title = result['title']
+        t_len = len(title)
+        if t_len > 5 and t_len % 2 != 0:
+            t_len = t_len // 2
+            if title[t_len] == '-' and title[:t_len] == title[t_len+1:]:
+                result['title'] = title[:tlen]
+        return result

From fbc240cfdbe0067347d5d22e9924e2a6206ae7c9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 4 Jul 2022 23:25:56 +0100
Subject: [PATCH 314/394] Import compat_urlparse

---
 youtube_dl/extractor/thisvid.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index b97cd48e827..9e208586e68 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -7,6 +7,7 @@
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_urlparse,
 )
 from ..utils import (
     clean_html,

From 3f5a585e32410b6e2c052180f7141f5237d90061 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 5 Jul 2022 00:35:18 +0200
Subject: [PATCH 315/394] Update youtube_dl/extractor/thisvid.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 9e208586e68..6ee9049073a 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -173,7 +173,7 @@ def entries(page_url, html=None):
                     # member list page
                     next_page = urljoin(url, self._search_regex(
                         r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?:(?!\1).)+)''',
-                        next_page, 'next page link', group='url', default=None)
+                        next_page, 'next page link', group='url', default=None))
                 # in case a member page should have pagination-next with empty link, not just `else:`
                 if next_page is None:
                     # playlist page

From 6fd470cd3c868586078cf212dd7aedb85040ea3b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 4 Jul 2022 23:38:36 +0100
Subject: [PATCH 316/394] Missing quote

---
 youtube_dl/extractor/thisvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 6ee9049073a..5da508dba9a 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -210,7 +210,7 @@ class ThisVidPlaylistIE(ThisVidMemberIE):
             'ext': 'mp4',
             'title': 'Big Italian Booty 28',
         'params': {
-            'noplaylist: True,
+            'noplaylist': True,
         },
     }]
 

From 883a0f83e76c61e5a5d7e0f4ab8c2bdf6f84d677 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Tue, 5 Jul 2022 00:41:06 +0200
Subject: [PATCH 317/394] Update thisvid.py

---
 youtube_dl/extractor/thisvid.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 5da508dba9a..7a2292d4682 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -209,6 +209,7 @@ class ThisVidPlaylistIE(ThisVidMemberIE):
             'id': '1072387',
             'ext': 'mp4',
             'title': 'Big Italian Booty 28',
+        },
         'params': {
             'noplaylist': True,
         },

From c86eb87916a9009fa4cc0415f46dc242acf4f8d3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 4 Jul 2022 23:51:49 +0100
Subject: [PATCH 318/394] Linter

---
 youtube_dl/extractor/thisvid.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
index 7a2292d4682..80069ac7694 100644
--- a/youtube_dl/extractor/thisvid.py
+++ b/youtube_dl/extractor/thisvid.py
@@ -12,6 +12,7 @@
 from ..utils import (
     clean_html,
     get_element_by_class,
+    int_or_none,
     sanitize_url,
     url_or_none,
     urljoin,
@@ -224,7 +225,6 @@ def _real_extract(self, url):
 
         self.to_screen(
             'Downloading playlist {pl_id} - add --no-playlist to download just the featured video').format(locals())
-        
         result = super(ThisVidPlaylistIE, self)._real_extract(url)
 
         # rework title returned as `the title - the title`
@@ -232,6 +232,6 @@ def _real_extract(self, url):
         t_len = len(title)
         if t_len > 5 and t_len % 2 != 0:
             t_len = t_len // 2
-            if title[t_len] == '-' and title[:t_len] == title[t_len+1:]:
-                result['title'] = title[:tlen]
+            if title[t_len] == '-' and title[:t_len] == title[t_len + 1:]:
+                result['title'] = title[:t_len]
         return result

From 129cfb174c8cc98d912a0565fb8c94c5f29f6c7e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 8 Jul 2022 06:54:01 +0100
Subject: [PATCH 319/394] Add Unicode compatibility header

---
 youtube_dl/extractor/livestreamfails.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index b92eca04700..64fed1bed71 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 import json
 import time

From dce1e146109d60949e5660ea134fcec95441438e Mon Sep 17 00:00:00 2001
From: Ehtisham Sabir <Shaamdbz@gmail.com>
Date: Sat, 9 Jul 2022 15:07:13 +0500
Subject: [PATCH 320/394] fixed issues.

---
 youtube_dl/extractor/wikimedia.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wikimedia.py b/youtube_dl/extractor/wikimedia.py
index 364c77a33d0..bb05682ffe1 100644
--- a/youtube_dl/extractor/wikimedia.py
+++ b/youtube_dl/extractor/wikimedia.py
@@ -28,13 +28,13 @@ class WikimediaIE(InfoExtractor):
             'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
             'license': 'md5:62907cddf705a9f7ae7076c15407a977',
             'author': None, 'subtitles': {'de': [{'ext': 'srt',
-                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=de&amp;trackformat=vtt '}],
+                                                  'url': 'https?://commons.wikimedia.org/w/api.php'}],
                                           'en-gb': [{'ext': 'srt',
-                                                     'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=en-gb&amp;trackformat=vtt '}],
+                                                     'url': 'https?://commons.wikimedia.org/w/api.php'}],
                                           'nl': [{'ext': 'srt',
-                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS.webm&amp;lang=nl&amp;trackformat=vtt '}],
+                                                  'url': 'https?://commons.wikimedia.org/w/api.php'}],
                                           'en': [{'ext': 'srt',
-                                                  'url': 'https://commons.wikimedia.org/w/api.php?action=timedtext&amp;title=File%3ADie_Temperaturkurve_der_Erde_%28ZDF%2C_Terra_X%29_720p_HD_50FPS_-_redub_NL.webm&amp;lang=en&amp;trackformat=vtt '}]}}
+                                                  'url': 're:https?://commons.wikimedia.org/w/api.php'}]}}
     }
 
     def _real_extract(self, url):

From 2d82e8a9587d99b1ca24f963ce1f9b449076e057 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:32:33 +0200
Subject: [PATCH 321/394] Refactor return

---
 youtube_dl/extractor/livestreamfails.py | 34 ++++++++++++-------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index 64fed1bed71..111cc19a129 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -25,28 +25,26 @@ class LivestreamfailsIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        result = {}
-        result['id'] = self._match_id(url)
+        id = self._match_id(url)
 
         # https://livestreamfails.com/clip/id uses https://api.livestreamfails.com/clip/ to fetch the video metadata
         # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
-        apiResponse = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + result['id'], result['id']))
-
-        # Twitch ID of clip
-        result['display_id'] = apiResponse.get('sourceId')
+        apiResponse = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + id, id))
 
         # Get the input timestamp (test case gives 2022-06-26T19:29:45.515Z)
-        result['timestamp'] = apiResponse.get('createdAt')
-        if(result.get('timestamp')):
+        timestamp = apiResponse.get('createdAt')
+        if(timestamp):
             # Parse it into a struct_time
-            result['timestamp'] = time.strptime(result['timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            timestamp = time.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
             # Convert the struct_time to a UNIX timestamp while ignoring the local timezone attached by time.strptime()
-            result['timestamp'] = calendar.timegm(result['timestamp'])
-
-        # Other fields
-        result['url'] = 'https://livestreamfails-video-prod.b-cdn.net/video/' + apiResponse.get('videoId')
-        result['title'] = apiResponse.get('label')
-        result['creator'] = apiResponse.get('streamer', {}).get('label')
-        result['thumbnail'] = 'https://livestreamfails-image-prod.b-cdn.net/image/' + apiResponse.get('imageId')
-
-        return result
+            timestamp = calendar.timegm(timestamp)
+
+        return {
+            'id': id,
+            'display_id': apiResponse.get('sourceId'),  # Twitch ID of clip
+            'timestamp': timestamp,
+            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + apiResponse.get('videoId'),
+            'title': apiResponse.get('label'),
+            'creator': apiResponse.get('streamer', {}).get('label'),
+            'thumbnail': 'https://livestreamfails-image-prod.b-cdn.net/image/' + apiResponse.get('imageId'),
+        }

From c9e09d187f0ce8729ec4f877ec9e051dd0ccb7a0 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:33:21 +0200
Subject: [PATCH 322/394] Rename apiResponse to api_response

---
 youtube_dl/extractor/livestreamfails.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index 111cc19a129..da24726012c 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -29,10 +29,10 @@ def _real_extract(self, url):
 
         # https://livestreamfails.com/clip/id uses https://api.livestreamfails.com/clip/ to fetch the video metadata
         # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
-        apiResponse = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + id, id))
+        api_response = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + id, id))
 
         # Get the input timestamp (test case gives 2022-06-26T19:29:45.515Z)
-        timestamp = apiResponse.get('createdAt')
+        timestamp = api_response.get('createdAt')
         if(timestamp):
             # Parse it into a struct_time
             timestamp = time.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
@@ -41,10 +41,10 @@ def _real_extract(self, url):
 
         return {
             'id': id,
-            'display_id': apiResponse.get('sourceId'),  # Twitch ID of clip
+            'display_id': api_response.get('sourceId'),  # Twitch ID of clip
             'timestamp': timestamp,
-            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + apiResponse.get('videoId'),
-            'title': apiResponse.get('label'),
-            'creator': apiResponse.get('streamer', {}).get('label'),
-            'thumbnail': 'https://livestreamfails-image-prod.b-cdn.net/image/' + apiResponse.get('imageId'),
+            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response.get('videoId'),
+            'title': api_response.get('label'),
+            'creator': api_response.get('streamer', {}).get('label'),
+            'thumbnail': 'https://livestreamfails-image-prod.b-cdn.net/image/' + api_response.get('imageId'),
         }

From 26dc85adb44091139b0273bec7a525c7e1313e41 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:35:21 +0200
Subject: [PATCH 323/394] Use IE._download_json()

---
 youtube_dl/extractor/livestreamfails.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index da24726012c..22180d9867e 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-import json
 import time
 import calendar
 
@@ -29,7 +28,7 @@ def _real_extract(self, url):
 
         # https://livestreamfails.com/clip/id uses https://api.livestreamfails.com/clip/ to fetch the video metadata
         # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
-        api_response = json.loads(self._download_webpage('https://api.livestreamfails.com/clip/' + id, id))
+        api_response = self._download_json('https://api.livestreamfails.com/clip/' + id, id)
 
         # Get the input timestamp (test case gives 2022-06-26T19:29:45.515Z)
         timestamp = api_response.get('createdAt')

From ea4a9f4d9868fb99aea2d8e0755c23af509a53cf Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:45:51 +0200
Subject: [PATCH 324/394] Correct which fields are fail inducing and crash
 proof

---
 youtube_dl/extractor/livestreamfails.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index 22180d9867e..04d533e0515 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -4,6 +4,11 @@
 from .common import InfoExtractor
 import time
 import calendar
+from ..utils import (
+    try_get,
+    url_or_none,
+)
+from ..compat import compat_str
 
 
 class LivestreamfailsIE(InfoExtractor):
@@ -40,10 +45,10 @@ def _real_extract(self, url):
 
         return {
             'id': id,
+            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response['videoId'],
+            'title': api_response['label'],
             'display_id': api_response.get('sourceId'),  # Twitch ID of clip
             'timestamp': timestamp,
-            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response.get('videoId'),
-            'title': api_response.get('label'),
-            'creator': api_response.get('streamer', {}).get('label'),
-            'thumbnail': 'https://livestreamfails-image-prod.b-cdn.net/image/' + api_response.get('imageId'),
+            'creator': try_get(api_response, lambda x: x['streamer']['label'], compat_str),
+            'thumbnail': url_or_none(try_get(api_response, lambda x: 'https://livestreamfails-image-prod.b-cdn.net/image/' + x['imageId']))
         }

From 4d02ba27d7377a4313bf775f7720cd9e3268aba2 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:48:32 +0200
Subject: [PATCH 325/394] Use parse_iso8601

---
 youtube_dl/extractor/livestreamfails.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index 04d533e0515..acf486f74c6 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -7,6 +7,7 @@
 from ..utils import (
     try_get,
     url_or_none,
+    parse_iso8601,
 )
 from ..compat import compat_str
 
@@ -35,20 +36,12 @@ def _real_extract(self, url):
         # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
         api_response = self._download_json('https://api.livestreamfails.com/clip/' + id, id)
 
-        # Get the input timestamp (test case gives 2022-06-26T19:29:45.515Z)
-        timestamp = api_response.get('createdAt')
-        if(timestamp):
-            # Parse it into a struct_time
-            timestamp = time.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
-            # Convert the struct_time to a UNIX timestamp while ignoring the local timezone attached by time.strptime()
-            timestamp = calendar.timegm(timestamp)
-
         return {
             'id': id,
             'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response['videoId'],
             'title': api_response['label'],
             'display_id': api_response.get('sourceId'),  # Twitch ID of clip
-            'timestamp': timestamp,
+            'timestamp': parse_iso8601(api_response.get('createdAt')),
             'creator': try_get(api_response, lambda x: x['streamer']['label'], compat_str),
             'thumbnail': url_or_none(try_get(api_response, lambda x: 'https://livestreamfails-image-prod.b-cdn.net/image/' + x['imageId']))
         }

From 918113db3b7e66b3c450345a436379e87150afe7 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 01:33:32 +0000
Subject: [PATCH 326/394] Break out video_url and title

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/livestreamfails.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index acf486f74c6..aabc7ecd291 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -36,10 +36,13 @@ def _real_extract(self, url):
         # Use the same endpoint here to avoid loading and parsing the provided page (which requires JS)
         api_response = self._download_json('https://api.livestreamfails.com/clip/' + id, id)
 
+        video_url = 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response['videoId']
+        title = api_response['label']
+
         return {
             'id': id,
-            'url': 'https://livestreamfails-video-prod.b-cdn.net/video/' + api_response['videoId'],
-            'title': api_response['label'],
+            'url': video_url,
+            'title': title,
             'display_id': api_response.get('sourceId'),  # Twitch ID of clip
             'timestamp': parse_iso8601(api_response.get('createdAt')),
             'creator': try_get(api_response, lambda x: x['streamer']['label'], compat_str),

From 032aed0e413b20f4fc62a6311834516e2a328cd1 Mon Sep 17 00:00:00 2001
From: nomevi <108267441+nomevi@users.noreply.github.com>
Date: Sun, 10 Jul 2022 03:33:15 +0200
Subject: [PATCH 327/394] Remove unused includes

---
 youtube_dl/extractor/livestreamfails.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/livestreamfails.py b/youtube_dl/extractor/livestreamfails.py
index aabc7ecd291..05f2cdcace7 100644
--- a/youtube_dl/extractor/livestreamfails.py
+++ b/youtube_dl/extractor/livestreamfails.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-import time
-import calendar
 from ..utils import (
     try_get,
     url_or_none,

From 5a6ec5ac4cb43d71f3fcd8e88358d1e7e0704f39 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 10 Jul 2022 18:20:00 +0200
Subject: [PATCH 328/394] cleaning

---
 README.md              | 6 ++++--
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 153f40b946e..cd2d36993f4 100644
--- a/README.md
+++ b/README.md
@@ -142,9 +142,11 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          download the videos at index 1, 2, 3,
                                          7, 10, 11, 12 and 13.
     --match-title REGEX                  Download only matching titles (case-
-                                         insensitive regex or sub-string)
+                                         insensitive regex or alphanumeric sub-
+                                         string)
     --reject-title REGEX                 Skip download for matching titles
-                                         (case-insensitive regex or sub-string)
+                                         (case-insensitive regex or alphanumeric
+                                         sub-string)
     --max-downloads NUMBER               Abort after downloading NUMBER files
     --min-filesize SIZE                  Do not download any videos smaller than
                                          SIZE (e.g. 50k or 44.6m)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 34c67a6ea96..c1219e495a9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -489,6 +489,7 @@
  - **LiveJournal**
  - **livestream**
  - **livestream:original**
+ - **Livestreamfails**
  - **LnkGo**
  - **loc**: Library of Congress
  - **LocalNews8**
@@ -991,6 +992,9 @@
  - **ThisAmericanLife**
  - **ThisAV**
  - **ThisOldHouse**
+ - **ThisVid**
+ - **ThisVidMember**
+ - **ThisVidPlaylist**
  - **tiktok**: TikTok video extractor
  - **tiktok:shortcut**: Resolver for TikTok shortcuts
  - **tiktok:user**: TikTok user profile extractor
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6bf47e50497..14666ef585c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.06.30'
+__version__ = '2022.07.10'

From 41f5fd818b75e4a119854065ed3d05daaef53c93 Mon Sep 17 00:00:00 2001
From: Niyazi Suleymanov <niyazisuleymanov@protonmail.com>
Date: Sat, 16 Jul 2022 22:52:19 +0400
Subject: [PATCH 329/394] entry no longer exists

---
 youtube_dl/extractor/newgrounds.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 82e7cf52216..f65843358e2 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -26,17 +26,6 @@ class NewgroundsIE(InfoExtractor):
             'upload_date': '20130911',
             'duration': 143,
         },
-    }, {
-        'url': 'https://www.newgrounds.com/portal/view/673111',
-        'md5': '3394735822aab2478c31b1004fe5e5bc',
-        'info_dict': {
-            'id': '673111',
-            'ext': 'mp4',
-            'title': 'Dancin',
-            'uploader': 'Squirrelman82',
-            'timestamp': 1460256780,
-            'upload_date': '20160410',
-        },
     }, {
         # source format unavailable, additional mp4 formats
         'url': 'http://www.newgrounds.com/portal/view/689400',

From 9036cf5132215645248ce804211e43974b7e4952 Mon Sep 17 00:00:00 2001
From: Niyazi Suleymanov <niyazisuleymanov@protonmail.com>
Date: Sat, 16 Jul 2022 23:11:50 +0400
Subject: [PATCH 330/394] add new test

---
 youtube_dl/extractor/newgrounds.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index f65843358e2..3f4a5d11f1c 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -26,6 +26,17 @@ class NewgroundsIE(InfoExtractor):
             'upload_date': '20130911',
             'duration': 143,
         },
+    }, {
+        'url': 'https://www.newgrounds.com/portal/view/850292',
+        'md5': 'bb7cacf45e1b4d648e2dac2d79284d67',
+        'info_dict': {
+            'id': '850292',
+            'ext': 'mp4',
+            'title': 'Timeless (2021)',
+            'uploader': 'Kevuhn',
+            'timestamp': 1657896960,
+            'upload_date': '20220715',
+        },
     }, {
         # source format unavailable, additional mp4 formats
         'url': 'http://www.newgrounds.com/portal/view/689400',
@@ -44,7 +55,7 @@ class NewgroundsIE(InfoExtractor):
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
-
+        
         webpage = self._download_webpage(url, media_id)
 
         title = self._html_search_regex(
@@ -80,7 +91,6 @@ def _real_extract(self, url):
             (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
              r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
             fatal=False)
-
         timestamp = unified_timestamp(self._html_search_regex(
             (r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+</dd>\s*<dd>[^<]+)',
              r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+)'), webpage, 'timestamp',

From 45ae186416881733ffe8d742107e748e06699948 Mon Sep 17 00:00:00 2001
From: Niyazi Suleymanov <niyazisuleymanov@protonmail.com>
Date: Sun, 17 Jul 2022 01:20:01 +0400
Subject: [PATCH 331/394] change incorrect uploader

---
 youtube_dl/extractor/newgrounds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 3f4a5d11f1c..9cb794ec89a 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -44,7 +44,7 @@ class NewgroundsIE(InfoExtractor):
             'id': '689400',
             'ext': 'mp4',
             'title': 'ZTV News Episode 8',
-            'uploader': 'BennettTheSage',
+            'uploader': 'ZONE-SAMA',
             'timestamp': 1487965140,
             'upload_date': '20170224',
         },

From f6c4b9da47423bbecc0ef0f60107ffac9e0fbb09 Mon Sep 17 00:00:00 2001
From: Niyazi Suleymanov <niyazisuleymanov@protonmail.com>
Date: Sun, 17 Jul 2022 01:27:56 +0400
Subject: [PATCH 332/394] fix #30575

---
 youtube_dl/extractor/newgrounds.py | 77 +++++++++++++++++-------------
 1 file changed, 43 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 9cb794ec89a..d6ec468bce1 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -5,7 +5,6 @@
 from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
-    int_or_none,
     parse_duration,
     parse_filesize,
     unified_timestamp,
@@ -55,42 +54,50 @@ class NewgroundsIE(InfoExtractor):
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
-        
+
         webpage = self._download_webpage(url, media_id)
+      
+        paths = url.split('/')
+        if paths[-3] == 'audio':
+            isAudio = True
+        else:
+            isAudio = False 
+        
+        if isAudio:
+            media_url = self._parse_json(self._search_regex(
+                r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
+
+            uploader = self._html_search_regex(
+                (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
+                 r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
+                fatal=False)
+            
+            formats = [{
+                'url': media_url,
+                'format_id': 'source',
+                'quality': 1,
+            }]
+
+        else:
+            media_url = f'https://www.newgrounds.com/portal/video/{media_id}'
+            media = self._download_json(media_url, media_id, headers={'X-Requested-With': 'XMLHttpRequest'})
+
+            uploader = media['author']
+
+            sources = media['sources']
+
+            formats = []
+            for source in sources:
+                for i in range(len(sources[source])):
+                    formats.append({
+                        'url': sources[source][i]['src'],
+                        'format_id': source,
+                        'height': int(source[:-2]) # 1080p -> 1080
+                    })
+
 
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
-
-        media_url = self._parse_json(self._search_regex(
-            r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
-
-        formats = [{
-            'url': media_url,
-            'format_id': 'source',
-            'quality': 1,
-        }]
-
-        max_resolution = int_or_none(self._search_regex(
-            r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
-            default=None))
-        if max_resolution:
-            url_base = media_url.rpartition('.')[0]
-            for resolution in (360, 720, 1080):
-                if resolution > max_resolution:
-                    break
-                formats.append({
-                    'url': '%s.%dp.mp4' % (url_base, resolution),
-                    'format_id': '%dp' % resolution,
-                    'height': resolution,
-                })
-
-        self._check_formats(formats, media_id)
-        self._sort_formats(formats)
-
-        uploader = self._html_search_regex(
-            (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
-             r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
-            fatal=False)
         timestamp = unified_timestamp(self._html_search_regex(
             (r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+</dd>\s*<dd>[^<]+)',
              r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+)'), webpage, 'timestamp',
@@ -98,7 +105,6 @@ def _real_extract(self, url):
         duration = parse_duration(self._search_regex(
             r'(?s)<dd>\s*Song\s*</dd>\s*<dd>.+?</dd>\s*<dd>([^<]+)', webpage,
             'duration', default=None))
-
         filesize_approx = parse_filesize(self._html_search_regex(
             r'(?s)<dd>\s*Song\s*</dd>\s*<dd>(.+?)</dd>', webpage, 'filesize',
             default=None))
@@ -108,6 +114,9 @@ def _real_extract(self, url):
         if '<dd>Song' in webpage:
             formats[0]['vcodec'] = 'none'
 
+        self._check_formats(formats, media_id)
+        self._sort_formats(formats)
+        
         return {
             'id': media_id,
             'title': title,

From c848538aff38423d26cff9163aa984156370eea0 Mon Sep 17 00:00:00 2001
From: Niyazi Suleymanov <niyazisuleymanov@protonmail.com>
Date: Sun, 17 Jul 2022 01:51:53 +0400
Subject: [PATCH 333/394] follow coding conventions

---
 youtube_dl/extractor/newgrounds.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index d6ec468bce1..da06316aaad 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -56,13 +56,13 @@ def _real_extract(self, url):
         media_id = self._match_id(url)
 
         webpage = self._download_webpage(url, media_id)
-      
+
         paths = url.split('/')
         if paths[-3] == 'audio':
             isAudio = True
         else:
-            isAudio = False 
-        
+            isAudio = False
+
         if isAudio:
             media_url = self._parse_json(self._search_regex(
                 r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
@@ -71,7 +71,7 @@ def _real_extract(self, url):
                 (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
                  r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
                 fatal=False)
-            
+
             formats = [{
                 'url': media_url,
                 'format_id': 'source',
@@ -92,10 +92,9 @@ def _real_extract(self, url):
                     formats.append({
                         'url': sources[source][i]['src'],
                         'format_id': source,
-                        'height': int(source[:-2]) # 1080p -> 1080
+                        'height': int(source[:-2])  # 1080p -> 1080
                     })
 
-
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
         timestamp = unified_timestamp(self._html_search_regex(
@@ -116,7 +115,7 @@ def _real_extract(self, url):
 
         self._check_formats(formats, media_id)
         self._sort_formats(formats)
-        
+
         return {
             'id': media_id,
             'title': title,

From 3091daa8345566b6b7b26d9b505e6fde01059333 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 21 Jul 2022 21:18:56 +0100
Subject: [PATCH 334/394] Update youtube_dl/extractor/newgrounds.py - remove
 f-string

---
 youtube_dl/extractor/newgrounds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index da06316aaad..46a6b022561 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -79,7 +79,7 @@ def _real_extract(self, url):
             }]
 
         else:
-            media_url = f'https://www.newgrounds.com/portal/video/{media_id}'
+            media_url = 'https://www.newgrounds.com/portal/video/' + media_id
             media = self._download_json(media_url, media_id, headers={'X-Requested-With': 'XMLHttpRequest'})
 
             uploader = media['author']

From e8aca87278f43ddd577d57b6955eaa609aa5760e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 22 Jul 2022 15:48:26 +0100
Subject: [PATCH 335/394] Improve fallback

---
 youtube_dl/extractor/megacartoons.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index ea6161b7bab..672bccbdd1d 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from ..utils import (
@@ -10,8 +9,10 @@
     merge_dicts,
     NO_DEFAULT,
     RegexNotFoundError,
+    try_get,
     url_or_none,
 )
+from ..compat import compat_str
 
 from .common import InfoExtractor
 
@@ -82,9 +83,9 @@ def _real_extract(self, url):
 
         if 'url' not in info or 'thumbnail' not in info:
             # Video data is stored in a json -> extract it from the raw html
-            url_json = json.loads(self._html_search_regex(r'<div.*data-item=["/\'](?P<videourls>{.*})["/\'].*>', webpage, 'videourls'))
+            url_json = self._parse_json(self._html_search_regex(r'''<div\b[^>]+\bdata-item\s*=\s*(["'])(?P<videourls>\{.*})\1''', webpage, 'videourls', group='videourls', default='{}'), video_id, fatal=False)) or {}
 
-            video_url = url_or_none(url_json.get('sources')[0].get('src') or self._og_search_video_url(webpage))   # Get the video url
+            video_url = url_or_none(try_get(url_json, lambda x: x['sources'][0]['src'], compat_str) or self._og_search_video_url(webpage))   # Get the video url
             video_thumbnail = url_or_none(url_json.get('splash') or self._og_search_thumbnail(webpage))            # Get the thumbnail
             info = merge_dicts(info, {
                 'url': video_url,

From 2804216e81de31ea33f65de3881f68f28ae54a59 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 22 Jul 2022 15:54:30 +0100
Subject: [PATCH 336/394] Improve fallback

---
 youtube_dl/extractor/megacartoons.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megacartoons.py b/youtube_dl/extractor/megacartoons.py
index 672bccbdd1d..c6186e3f828 100644
--- a/youtube_dl/extractor/megacartoons.py
+++ b/youtube_dl/extractor/megacartoons.py
@@ -83,7 +83,7 @@ def _real_extract(self, url):
 
         if 'url' not in info or 'thumbnail' not in info:
             # Video data is stored in a json -> extract it from the raw html
-            url_json = self._parse_json(self._html_search_regex(r'''<div\b[^>]+\bdata-item\s*=\s*(["'])(?P<videourls>\{.*})\1''', webpage, 'videourls', group='videourls', default='{}'), video_id, fatal=False)) or {}
+            url_json = self._parse_json(self._html_search_regex(r'''<div\b[^>]+\bdata-item\s*=\s*(["'])(?P<videourls>\{.*})\1''', webpage, 'videourls', group='videourls', default='{}'), video_id, fatal=False) or {}
 
             video_url = url_or_none(try_get(url_json, lambda x: x['sources'][0]['src'], compat_str) or self._og_search_video_url(webpage))   # Get the video url
             video_thumbnail = url_or_none(url_json.get('splash') or self._og_search_thumbnail(webpage))            # Get the thumbnail

From 5b217541977d67a0b823cd224326339af88adbbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartosz=20Bia=C5=82as?= <bartosz.bialas@epfl.ch>
Date: Wed, 8 Jun 2022 19:11:24 +0200
Subject: [PATCH 337/394] [SwitchTube] Add new extractor

---
 docs/supportedsites.md             |   3 +
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/switchtube.py | 219 +++++++++++++++++++++++++++++
 3 files changed, 227 insertions(+)
 create mode 100644 youtube_dl/extractor/switchtube.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ae2a6b8b02b..6a3bab2426f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -910,6 +910,9 @@
  - **SVTPage**
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SVTSeries**
+ - **switchtube**
+ - **switchtube:channel**
+ - **switchtube:profile**
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..78ea0d9a48d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1209,6 +1209,11 @@
     SVTPlayIE,
     SVTSeriesIE,
 )
+from .switchtube import (
+  SwitchTubeIE,
+  SwitchTubeProfileIE,
+  SwitchTubeChannelIE,
+)
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/switchtube.py b/youtube_dl/extractor/switchtube.py
new file mode 100644
index 00000000000..3601572b6e0
--- /dev/null
+++ b/youtube_dl/extractor/switchtube.py
@@ -0,0 +1,219 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    extract_attributes,
+    float_or_none,
+    get_element_by_attribute,
+    int_or_none,
+    parse_iso8601,
+    strip_or_none,
+    urljoin,
+)
+
+
+class SwitchTubeIE(InfoExtractor):
+    _VALID_URL = r'https?://tube\.switch\.ch/videos/(?P<id>[\da-zA-Z]+)'
+    IE_NAME = 'switchtube'
+    _TESTS = [{
+        'url': 'https://tube.switch.ch/videos/0T1XfaIFSX',
+        'info_dict': {
+            'id': '0T1XfaIFSX',
+            'title': '2016_ASE_sqC03-Entretien',
+            'channel': 'ASE Assistant-e socio-éducatif-ve CFC',
+            'channel_url': 'https://tube.switch.ch/channels/bsaer76yoL',
+            'channel_id': 'bsaer76yoL',
+            'ext': 'mp4',
+            'description': None,
+            'thumbnail': r're:^https?://tube.switch.ch/image/representations/[\w-]+$',
+            'license': 'All rights reserved',
+            'creator': 'Jean-Marc Pouly from Eidgenössische Hochschule für Berufsbildung',
+            'uploader': 'Jean-Marc Pouly from Eidgenössische Hochschule für Berufsbildung',
+            'uploader_url': 'https://tube.switch.ch/profiles/42481',
+            'uploader_id': '42481',
+            'upload_date': '20220309',
+            'timestamp': 1646839068,
+        }
+    }, {
+        'url': 'https://tube.switch.ch/videos/0cf3886d',
+        'info_dict': {
+            'id': '0cf3886d',
+            'ext': 'mp4',
+            'title': 'Introduction: Mini-Batches in On- and Off-Policy Deep Reinforcement Learning',
+            'license': 'All rights reserved',
+            'description': 'One of the challenges in Deep Reinforcement Learning is to decorrelate the data. How this is possible with replay buffers is explained here.',
+            'thumbnail': r're:^https?://tube.switch.ch/image/representations/[\w-]+$',
+            'channel': 'CS-456 Artificial Neural Networks',
+            'channel_url': 'https://tube.switch.ch/channels/1deb03e0',
+            'channel_id': '1deb03e0',
+            'timestamp': 1590733406,
+            'upload_date': '20200529',
+            'creator': 'Wulfram Gerstner from École polytechnique fédérale de Lausanne (EPFL)',
+            'uploader': 'Wulfram Gerstner from École polytechnique fédérale de Lausanne (EPFL)',
+            'uploader_url': 'https://tube.switch.ch/profiles/94248',
+            'uploader_id': '94248',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title>', webpage, 'title')
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'is_live': False,
+        }
+
+        info['view_count'] = int_or_none(self._html_search_regex(r'(?s)<p\b[^>]*>\s*(\d+)\s+views?\s*</p>', webpage,
+                                                                 'view_count', default=None))
+        info['license'] = self._html_search_regex(r'''(?s)<span\b[^>]*?\bproperty\s*=\s*["'](?:\b[cd]c:license\b\s*){2,}[^>]+>(.+)</span>''',
+                                                  webpage, 'license', default=None)
+        info['description'] = strip_or_none(clean_html(
+            get_element_by_attribute('property', 'dc:description', webpage)))
+        info['duration'] = float_or_none(
+            self._search_regex(r'''\bdata-duration\s*=\s*["']([\d.]+)''',
+                               webpage, 'duration', default=None))
+
+        channel_groups = self._search_regex(r'''(?s)<a\b[^>]+?\bhref\s*=\s*["']/channels/(.{4,}?)</a>''',
+                                            webpage, 'channel groups', default='')
+        channel_groups = re.split(r'''(?s)["'][^>]*>\s*''', channel_groups, 1)
+        if len(channel_groups) == 2:
+            for i, ch in enumerate(('channel_id', 'channel')):
+                info[ch] = strip_or_none(channel_groups[i])
+            if info['channel_id']:
+                info['channel_url'] = 'https://tube.switch.ch/channels/' + info['channel_id']
+
+        def outer_elements_by_attribute(attr, value, html, tag=None, escape_value=True, include_inner=False):
+            """generate matching HTML element strings
+               if include_inner, tuples of (element, content)"""
+            pattern = r'''(?s)(?P<element><(%s)\b[^>]+?\b%s\s*=\s*("|')%s\b[^>]+>)(?P<inner>.*?)</\2>''' % \
+                (re.escape(tag) if tag is not None else r'\w+', attr, re.escape(value) if escape_value else value)
+            matches = re.finditer(pattern, html)
+            for m in matches:
+                yield m.group('element', 'inner') if include_inner else m.group('element')
+
+        for dt in outer_elements_by_attribute('property', 'dc:date', webpage, tag='span'):
+            dt = extract_attributes(dt)
+            if dt.get('class') == 'dt-published':
+                info['timestamp'] = parse_iso8601(dt.get('content'))
+                break
+
+        creator_groups = self._search_regex(
+            r'''(?s)<span\b[^>]+?\bclass\s*=\s*("|')(?:(?!\1).)*\bp-author\b(?:(?!\1).)*\1\s*property\s*=\s*["']dc:creator\b[^>]+>\s*(.*?<span\b[^>]+?\bclass\s*=\s*["']p-name\b.*</span>).*?</span>''',
+            webpage, 'creator groups', default='', group=2)
+        creator_groups = re.match(r'''(?s)<a\b[^>]+?\bhref\s*=\s*("|')/profiles/(?P<profile_id>.+?)\1[^>]*>\s*<span\b[^>]+>\s*(?P<creator_name>.+?)\s*</span>[,\s]*<span\b[^>]+\bclass\s*=\s*["']p-organization-name\b[^>]+>\s*(?P<organization_name>.+?)\s*</span>''', creator_groups)
+        if creator_groups:
+            creator_groups = creator_groups.groupdict()
+            info['uploader'] = info['creator'] = ' from '.join((creator_groups['creator_name'], creator_groups['organization_name']))
+            info['uploader_id'] = creator_groups['profile_id']
+            info['uploader_url'] = 'https://tube.switch.ch/profiles/' + info['uploader_id']
+
+        parsed_media_entries = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        info['thumbnail'] = parsed_media_entries['thumbnail']
+        info['formats'] = parsed_media_entries['formats']
+        self._sort_formats(info['formats'])
+
+        return info
+
+
+class SwitchTubeProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://tube\.switch\.ch/profiles/(?P<id>[\da-zA-Z]+)'
+    IE_NAME = 'switchtube:profile'
+    _TESTS = [{
+        'url': 'https://tube.switch.ch/profiles/94248',
+        'info_dict': {
+            'id': '94248',
+            'title': 'Wulfram Gerstner',
+            'description': None,
+        },
+        'playlist_mincount': 94,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if SwitchTubeIE.suitable(url) else super(SwitchTubeProfileIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        webpage = self._download_webpage(url, channel_id)
+        channel_title = self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title>', webpage, 'title', default=None)
+
+        description = self._html_search_regex(r'''(?s)<div\b[^>]+class\s*=\s*("|')p-summary formatted\1[^>]+property\s*=\s*("|')dc:description\2[^>]*>\s*(.+?)\s*</div>''',
+                                              webpage, 'description', default=None, group=3)
+
+        entries = []
+        next_page = None
+        for current_page_number in itertools.count(1):
+            if next_page:
+                webpage = self._download_webpage(next_page, channel_id, note='Downloading page %d' % (current_page_number, ))
+
+            for _, video_path, video_id, title in re.findall(
+                    r'''(?s)<a\b[^>]+\bhref\s*=\s*("|')(/videos/((?:(?!\1).)+?))\1[^>]*>\s*<div\b[^>]+\bclass\s*=\s*["']title\b[^>]+>(.+?)</div>''',
+                    webpage):
+                video_url = urljoin(url, video_path)
+                if video_url:
+                    entries.append(self.url_result(video_url, ie=SwitchTubeIE.ie_key(), video_id=video_id))
+
+            next_page = self._search_regex(
+                r'''<a\b[^>]+?\bhref\s*=\s*("|')(?P<path>/profiles/%s\?(?:(?!\1).)+)\1[^>]*>\s*Next\s*</a>''' % (channel_id,),
+                webpage, 'next page', group='path', default=None)
+            if next_page:
+                next_page = urljoin(url, next_page)
+            if not next_page:
+                break
+
+        return self.playlist_result(entries, channel_id, channel_title,
+                                    description)
+
+
+class SwitchTubeChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://tube\.switch\.ch/channels/(?P<id>[\da-zA-Z]+)'
+    IE_NAME = 'switchtube:channel'
+    _TESTS = [{
+        'url': 'https://tube.switch.ch/channels/1deb03e0',
+        'info_dict': {
+            'id': '1deb03e0',
+            'title': 'CS-456 Artificial Neural Networks',
+            'description': 'Class on Artificial Neural Networks and Reinforcement Learning designed for EPFL master students in CS and related disciplines.'
+        },
+        'playlist_mincount': 94,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if SwitchTubeIE.suitable(url) else super(SwitchTubeChannelIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        entries = []
+        for current_page_number in itertools.count(0):
+            page_url = urljoin(url, '/channels/%s?order=episodes&page=%d' % (channel_id, current_page_number))
+            webpage = self._download_webpage(page_url, channel_id, note='Downloading page %d' % (current_page_number + 1, ))
+
+            if current_page_number == 0:
+                channel_title = self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title>', webpage, 'title', default=None)
+                description = self._html_search_regex(r'''(?s)<div\b[^>]+class\s*=\s*("|')description formatted\1[^>]*>\s*<p>\s*(.+?)\s*</p>\s*</div>''',
+                                                      webpage, 'description', default=None, group=2)
+
+            this_page_still_has_something = False
+            for _, video_path, video_id in re.findall(
+                    r'''(?s)<a\b[^>]+\bhref\s*=\s*("|')(/videos/((?:(?!\1).)+?))\1[^>]*>''',
+                    webpage):
+                video_url = urljoin(url, video_path)
+                if video_url:
+                    this_page_still_has_something = True
+                    entries.append(self.url_result(video_url, ie=SwitchTubeIE.ie_key(), video_id=video_id))
+
+            if not this_page_still_has_something:
+                break
+
+        return self.playlist_result(entries, channel_id, channel_title,
+                                    description)

From 28f94a2828b5a3531ca529842e47c72a591f4c54 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Sun, 7 Aug 2022 08:03:17 -0700
Subject: [PATCH 338/394] [truth] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/truth.py      | 78 ++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/truth.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..4ecc1643b13 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1295,6 +1295,7 @@
     TrovoVodIE,
 )
 from .trunews import TruNewsIE
+from .truth import TruthIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
new file mode 100644
index 00000000000..5a5311eeb60
--- /dev/null
+++ b/youtube_dl/extractor/truth.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import clean_html, int_or_none, unified_timestamp
+
+
+class TruthIE(InfoExtractor):
+    """Extract videos from posts on Donald Trump's truthsocial.com."""
+
+    _VALID_URL = r"https://truthsocial\.com/@[^/]+/posts/(?P<id>[0-9]+)"
+    _TESTS = [
+        {
+            "url": "https://truthsocial.com/@realDonaldTrump/posts/108779000807761862",
+            "md5": "4a5fb1470c192e493d9efd6f19e514d3",
+            "info_dict": {
+                "id": "108779000807761862",
+                "ext": "qt",
+                "title": "Untitled",
+                "timestamp": 1659835827,
+                "upload_date": "20220807",
+                "uploader": "Donald J. Trump",
+                "uploader_id": "realDonaldTrump",
+            },
+        },
+        {
+            "url": "https://truthsocial.com/@ProjectVeritasAction/posts/108618228543962049",
+            "md5": "fd47ba68933f9dce27accc52275be9c3",
+            "info_dict": {
+                "id": "108618228543962049",
+                "ext": "mp4",
+                "title": "RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  “Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe",
+                "timestamp": 1657382637,
+                "upload_date": "20220709",
+                "uploader": "Project Veritas Action",
+                "uploader_id": "ProjectVeritasAction",
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Get data from API
+        api_url = "https://truthsocial.com/api/v1/statuses/" + video_id
+        status = self._download_json(api_url, video_id)
+
+        # Pull out video
+        attachments = status["media_attachments"]
+        video = attachments[0]
+        url = video["url"]
+
+        # Pull out metadata
+        title = clean_html(status.get("content")).replace("\n", "") or "Untitled"
+        account = status.get("account") or {}
+        timestamp = unified_timestamp(status.get("created_at"))
+        uploader = account.get("display_name")
+        uploader_id = account.get("username")
+        uploader_url = (
+            "https://truthsocial.com/@" + uploader_id if uploader_id else None
+        )
+        repost_count = int_or_none(status.get("reblogs_count"))
+        like_count = int_or_none(status.get("favourites_count"))
+        comment_count = int_or_none(status.get("replies_count"))
+
+        return {
+            "id": video_id,
+            "url": url,
+            "title": title,
+            "timestamp": timestamp,
+            "uploader": uploader,
+            "uploader_id": uploader_id,
+            "uploader_url": uploader_url,
+            "repost_count": repost_count,
+            "like_count": like_count,
+            "comment_count": comment_count,
+        }

From 13077c0c96097c2ba6493ceb26eb04df0b5d7ce3 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Sun, 7 Aug 2022 08:17:19 -0700
Subject: [PATCH 339/394] Add _GEO_COUNTRIES = ['US']

---
 youtube_dl/extractor/truth.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index 5a5311eeb60..0f5b044f017 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -38,6 +38,7 @@ class TruthIE(InfoExtractor):
             },
         },
     ]
+    _GEO_COUNTRIES = ['US']  # The site is only available in the US
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From e9839174a16c4640387d91aff3ea0504dfdbeadc Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Sun, 7 Aug 2022 11:14:24 -0700
Subject: [PATCH 340/394] [parler] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/parler.py     | 74 ++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/parler.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..5c1cee28f25 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -908,6 +908,7 @@
     PalcoMP3VideoIE,
 )
 from .pandoratv import PandoraTVIE
+from .parler import ParlerIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
new file mode 100644
index 00000000000..8cd60765920
--- /dev/null
+++ b/youtube_dl/extractor/parler.py
@@ -0,0 +1,74 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from urllib import parse
+
+from .common import InfoExtractor
+
+from ..utils import clean_html, unified_timestamp
+
+
+class ParlerIE(InfoExtractor):
+    """Extract videos from posts on Parler."""
+
+    _VALID_URL = r"https://parler\.com/feed/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"
+    _TESTS = [
+        {
+            "url": "https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7",
+            "md5": "16e0f447bf186bb3cf64de5bbbf4d22d",
+            "info_dict": {
+                "id": "df79fdba-07cc-48fe-b085-3293897520d7",
+                "ext": "mp4",
+                "title": "Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.",
+                "timestamp": 1659744000,
+                "upload_date": "20220806",
+                "uploader": "Tulsi Gabbard",
+                "uploader_id": "TulsiGabbard",
+            },
+        },
+        {
+            "url": "https://parler.com/feed/a7406eb4-91e5-4793-b5e3-ade57a24e287",
+            "md5": "11687e2f5bb353682cee338d181422ed",
+            "info_dict": {
+                "id": "a7406eb4-91e5-4793-b5e3-ade57a24e287",
+                "ext": "mp4",
+                "title": "This man should run for office",
+                "timestamp": 1659657600,
+                "upload_date": "20220805",
+                "uploader": "Benny Johnson",
+                "uploader_id": "BennyJohnson",
+            },
+        },
+    ]
+    _GEO_COUNTRIES = ["US"]  # The site is only available in the US
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Get data from API
+        api_url = "https://parler.com/open-api/ParleyDetailEndpoint.php"
+        payload = parse.urlencode({"uuid": video_id}).encode()
+        status = self._download_json(api_url, video_id, data=payload)
+
+        # Pull out video
+        data = status["data"][0]["primary"]
+        video = data["video_data"]
+        url = video["videoSrc"]
+
+        # Pull out metadata
+        title = clean_html(data.get("full_body")).replace("\n", "")
+        timestamp = unified_timestamp(data.get("date_created"))
+        uploader = data.get("name")
+        uploader_id = data.get("username")
+        uploader_url = "https://parler.com/" + uploader_id if uploader_id else None
+
+        # Return the result
+        return {
+            "id": video_id,
+            "url": url,
+            "title": title,
+            "timestamp": timestamp,
+            "uploader": uploader,
+            "uploader_id": uploader_id,
+            "uploader_url": uploader_url,
+        }

From 6d6f38bed8062ee754af342fbfddb91b004555d8 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Sun, 7 Aug 2022 11:14:38 -0700
Subject: [PATCH 341/394] Cut geography

---
 youtube_dl/extractor/parler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 8cd60765920..e5660798107 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -40,7 +40,6 @@ class ParlerIE(InfoExtractor):
             },
         },
     ]
-    _GEO_COUNTRIES = ["US"]  # The site is only available in the US
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a6f8feef491dc4b0af0082b16788a1e28ef9c61a Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 09:45:59 -0700
Subject: [PATCH 342/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index e5660798107..ecc44216c68 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -5,7 +5,12 @@
 
 from .common import InfoExtractor
 
-from ..utils import clean_html, unified_timestamp
+from ..utils import (
+    clean_html,
+    strip_or_none,
+    unified_timestamp,
+    urlencode_postdata,
+)
 
 
 class ParlerIE(InfoExtractor):

From b1bbb03e0b8f026e78748f6bfd822691522ddf13 Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 09:46:26 -0700
Subject: [PATCH 343/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index ecc44216c68..0b09c195507 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -16,7 +16,8 @@
 class ParlerIE(InfoExtractor):
     """Extract videos from posts on Parler."""
 
-    _VALID_URL = r"https://parler\.com/feed/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"
+    _UUID_RE = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+    _VALID_URL = r'https://parler\.com/feed/(?P<id>%s)' % (_UUID_RE, )
     _TESTS = [
         {
             "url": "https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7",

From 1a82825b9c4ce8b88b3c5cf36742bbaba91e68ae Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 09:46:46 -0700
Subject: [PATCH 344/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 0b09c195507..ab7ddf31921 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -51,7 +51,7 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         # Get data from API
-        api_url = "https://parler.com/open-api/ParleyDetailEndpoint.php"
+        api_url = 'https://parler.com/open-api/ParleyDetailEndpoint.php'
         payload = parse.urlencode({"uuid": video_id}).encode()
         status = self._download_json(api_url, video_id, data=payload)
 

From ff952d586b3302d706cbdc14de8c17f75990397d Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 09:47:08 -0700
Subject: [PATCH 345/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index ab7ddf31921..b3cd3ab85e2 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -56,9 +56,9 @@ def _real_extract(self, url):
         status = self._download_json(api_url, video_id, data=payload)
 
         # Pull out video
-        data = status["data"][0]["primary"]
-        video = data["video_data"]
-        url = video["videoSrc"]
+        url = status['data'][0]['primary']['video_data']['videoSrc']
+        # now we know this exists and is a dict
+        data = status['data'][0]['primary']
 
         # Pull out metadata
         title = clean_html(data.get("full_body")).replace("\n", "")

From b14b4b8af50c5395777671f63212220d3ce1d582 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 8 Aug 2022 11:01:58 -0700
Subject: [PATCH 346/394] Single quotes

---
 youtube_dl/extractor/parler.py | 40 +++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index e5660798107..7cfd1c6cd43 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -14,29 +14,29 @@ class ParlerIE(InfoExtractor):
     _VALID_URL = r"https://parler\.com/feed/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"
     _TESTS = [
         {
-            "url": "https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7",
-            "md5": "16e0f447bf186bb3cf64de5bbbf4d22d",
-            "info_dict": {
-                "id": "df79fdba-07cc-48fe-b085-3293897520d7",
-                "ext": "mp4",
-                "title": "Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.",
-                "timestamp": 1659744000,
-                "upload_date": "20220806",
-                "uploader": "Tulsi Gabbard",
-                "uploader_id": "TulsiGabbard",
+            'url': 'https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7',
+            'md5': '16e0f447bf186bb3cf64de5bbbf4d22d',
+            'info_dict': {
+                'id': 'df79fdba-07cc-48fe-b085-3293897520d7',
+                'ext': 'mp4',
+                'title': 'Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.',
+                'timestamp': 1659744000,
+                'upload_date': '20220806',
+                'uploader': 'Tulsi Gabbard',
+                'uploader_id': 'TulsiGabbard',
             },
         },
         {
-            "url": "https://parler.com/feed/a7406eb4-91e5-4793-b5e3-ade57a24e287",
-            "md5": "11687e2f5bb353682cee338d181422ed",
-            "info_dict": {
-                "id": "a7406eb4-91e5-4793-b5e3-ade57a24e287",
-                "ext": "mp4",
-                "title": "This man should run for office",
-                "timestamp": 1659657600,
-                "upload_date": "20220805",
-                "uploader": "Benny Johnson",
-                "uploader_id": "BennyJohnson",
+            'url': 'https://parler.com/feed/a7406eb4-91e5-4793-b5e3-ade57a24e287',
+            'md5': '11687e2f5bb353682cee338d181422ed',
+            'info_dict': {
+                'id': 'a7406eb4-91e5-4793-b5e3-ade57a24e287',
+                'ext': 'mp4',
+                'title': 'This man should run for office',
+                'timestamp': 1659657600,
+                'upload_date': '20220805',
+                'uploader': 'Benny Johnson',
+                'uploader_id': 'BennyJohnson',
             },
         },
     ]

From 4b3bc81777bd57e26c2ce29800f76af7b8c5c12e Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 11:03:00 -0700
Subject: [PATCH 347/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 79888d8dd89..267d9ff6b2a 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -63,8 +63,8 @@ def _real_extract(self, url):
         # Pull out metadata
         title = clean_html(data.get("full_body")).replace("\n", "")
         timestamp = unified_timestamp(data.get("date_created"))
-        uploader = data.get("name")
-        uploader_id = data.get("username")
+        uploader = strip_or_none(data.get('name'))
+        uploader_id = strip_or_none(data.get('username'))
         uploader_url = "https://parler.com/" + uploader_id if uploader_id else None
 
         # Return the result

From 40f370c358fc374bc42aa721c76c6611a60ea69b Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 11:03:09 -0700
Subject: [PATCH 348/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 267d9ff6b2a..b35ff6cbddb 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -62,7 +62,7 @@ def _real_extract(self, url):
 
         # Pull out metadata
         title = clean_html(data.get("full_body")).replace("\n", "")
-        timestamp = unified_timestamp(data.get("date_created"))
+        timestamp = unified_timestamp(data.get('date_created'))
         uploader = strip_or_none(data.get('name'))
         uploader_id = strip_or_none(data.get('username'))
         uploader_url = "https://parler.com/" + uploader_id if uploader_id else None

From e46a2a58e200aa736eb5594c49b169848d2dbb78 Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 11:03:18 -0700
Subject: [PATCH 349/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index b35ff6cbddb..76033042a90 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -65,7 +65,7 @@ def _real_extract(self, url):
         timestamp = unified_timestamp(data.get('date_created'))
         uploader = strip_or_none(data.get('name'))
         uploader_id = strip_or_none(data.get('username'))
-        uploader_url = "https://parler.com/" + uploader_id if uploader_id else None
+        uploader_url = ('https://parler.com/' + uploader_id) if uploader_id else None
 
         # Return the result
         return {

From 92e053f4d26c6db854472f90d1a5f6e32fa1438a Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 8 Aug 2022 11:04:11 -0700
Subject: [PATCH 350/394] Single quotes

---
 youtube_dl/extractor/parler.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 79888d8dd89..388af38dccc 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -69,11 +69,11 @@ def _real_extract(self, url):
 
         # Return the result
         return {
-            "id": video_id,
-            "url": url,
-            "title": title,
-            "timestamp": timestamp,
-            "uploader": uploader,
-            "uploader_id": uploader_id,
-            "uploader_url": uploader_url,
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
         }

From fec554563ad653c97acb05437ffcb3fe6bc39482 Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 11:04:54 -0700
Subject: [PATCH 351/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 05c40841789..5217d71c824 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -52,7 +52,7 @@ def _real_extract(self, url):
 
         # Get data from API
         api_url = 'https://parler.com/open-api/ParleyDetailEndpoint.php'
-        payload = parse.urlencode({"uuid": video_id}).encode()
+        payload = urlencode_postdata({'uuid': video_id})
         status = self._download_json(api_url, video_id, data=payload)
 
         # Pull out video

From 2e1c744be831d8410aee7963715d16bfcf006b05 Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Mon, 8 Aug 2022 11:05:07 -0700
Subject: [PATCH 352/394] Update youtube_dl/extractor/parler.py

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 5217d71c824..b01f88e2e38 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -61,7 +61,7 @@ def _real_extract(self, url):
         data = status['data'][0]['primary']
 
         # Pull out metadata
-        title = clean_html(data.get("full_body")).replace("\n", "")
+        title = clean_html(data.get('full_body')) or self.generic_title(url)
         timestamp = unified_timestamp(data.get('date_created'))
         uploader = strip_or_none(data.get('name'))
         uploader_id = strip_or_none(data.get('username'))

From 1f5cc9275222f7579b149d32dcee976db0f59a3e Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 8 Aug 2022 11:39:35 -0700
Subject: [PATCH 353/394] Style fixes

---
 youtube_dl/extractor/truth.py | 96 ++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index 0f5b044f017..b4827377980 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -3,38 +3,43 @@
 
 from .common import InfoExtractor
 
-from ..utils import clean_html, int_or_none, unified_timestamp
+from ..utils import (
+    clean_html,
+    int_or_none,
+    unified_timestamp,
+    strip_or_none
+)
 
 
 class TruthIE(InfoExtractor):
     """Extract videos from posts on Donald Trump's truthsocial.com."""
 
-    _VALID_URL = r"https://truthsocial\.com/@[^/]+/posts/(?P<id>[0-9]+)"
+    _VALID_URL = r'https://truthsocial\.com/@[^/]+/posts/(?P<id>[0-9]+)'
     _TESTS = [
         {
-            "url": "https://truthsocial.com/@realDonaldTrump/posts/108779000807761862",
-            "md5": "4a5fb1470c192e493d9efd6f19e514d3",
-            "info_dict": {
-                "id": "108779000807761862",
-                "ext": "qt",
-                "title": "Untitled",
-                "timestamp": 1659835827,
-                "upload_date": "20220807",
-                "uploader": "Donald J. Trump",
-                "uploader_id": "realDonaldTrump",
+            'url': 'https://truthsocial.com/@realDonaldTrump/posts/108779000807761862',
+            'md5': '4a5fb1470c192e493d9efd6f19e514d3',
+            'info_dict': {
+                'id': '108779000807761862',
+                'ext': 'qt',
+                'title': '0d8691160c73d663',
+                'timestamp': 1659835827,
+                'upload_date': '20220807',
+                'uploader': 'Donald J. Trump',
+                'uploader_id': 'realDonaldTrump',
             },
         },
         {
-            "url": "https://truthsocial.com/@ProjectVeritasAction/posts/108618228543962049",
-            "md5": "fd47ba68933f9dce27accc52275be9c3",
-            "info_dict": {
-                "id": "108618228543962049",
-                "ext": "mp4",
-                "title": "RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  “Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe",
-                "timestamp": 1657382637,
-                "upload_date": "20220709",
-                "uploader": "Project Veritas Action",
-                "uploader_id": "ProjectVeritasAction",
+            'url': 'https://truthsocial.com/@ProjectVeritasAction/posts/108618228543962049',
+            'md5': 'fd47ba68933f9dce27accc52275be9c3',
+            'info_dict': {
+                'id': '108618228543962049',
+                'ext': 'mp4',
+                'title': """RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  \n“Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe""",
+                'timestamp': 1657382637,
+                'upload_date': '20220709',
+                'uploader': 'Project Veritas Action',
+                'uploader_id': 'ProjectVeritasAction',
             },
         },
     ]
@@ -44,36 +49,33 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         # Get data from API
-        api_url = "https://truthsocial.com/api/v1/statuses/" + video_id
+        api_url = 'https://truthsocial.com/api/v1/statuses/' + video_id
         status = self._download_json(api_url, video_id)
 
         # Pull out video
-        attachments = status["media_attachments"]
-        video = attachments[0]
-        url = video["url"]
+        url = status['media_attachments'][0]['url']
 
         # Pull out metadata
-        title = clean_html(status.get("content")).replace("\n", "") or "Untitled"
-        account = status.get("account") or {}
-        timestamp = unified_timestamp(status.get("created_at"))
-        uploader = account.get("display_name")
-        uploader_id = account.get("username")
-        uploader_url = (
-            "https://truthsocial.com/@" + uploader_id if uploader_id else None
-        )
-        repost_count = int_or_none(status.get("reblogs_count"))
-        like_count = int_or_none(status.get("favourites_count"))
-        comment_count = int_or_none(status.get("replies_count"))
+        title = strip_or_none(clean_html(status.get('content'))) or self._generic_title(url)
+        timestamp = unified_timestamp(status.get('created_at'))
+        account = status.get('account') or {}
+        uploader = strip_or_none(account.get('display_name'))
+        uploader_id = strip_or_none(account.get('username'))
+        uploader_url = ('https://truthsocial.com/@' + uploader_id) if uploader_id else None
+        repost_count = int_or_none(status.get('reblogs_count'))
+        like_count = int_or_none(status.get('favourites_count'))
+        comment_count = int_or_none(status.get('replies_count'))
 
+        # Return the stuff
         return {
-            "id": video_id,
-            "url": url,
-            "title": title,
-            "timestamp": timestamp,
-            "uploader": uploader,
-            "uploader_id": uploader_id,
-            "uploader_url": uploader_url,
-            "repost_count": repost_count,
-            "like_count": like_count,
-            "comment_count": comment_count,
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+            'repost_count': repost_count,
+            'like_count': like_count,
+            'comment_count': comment_count,
         }

From 871b60b117a10feb40e6f117e22a6736c4007d6f Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 8 Aug 2022 11:40:31 -0700
Subject: [PATCH 354/394] _generic_title

---
 youtube_dl/extractor/parler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index b01f88e2e38..b22de186ea4 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -61,7 +61,7 @@ def _real_extract(self, url):
         data = status['data'][0]['primary']
 
         # Pull out metadata
-        title = clean_html(data.get('full_body')) or self.generic_title(url)
+        title = clean_html(data.get('full_body')) or self._generic_title(url)
         timestamp = unified_timestamp(data.get('date_created'))
         uploader = strip_or_none(data.get('name'))
         uploader_id = strip_or_none(data.get('username'))

From 0ef718103f65062242fe8515cfe90f809fc5735f Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Tue, 9 Aug 2022 03:42:26 -0700
Subject: [PATCH 355/394] Shorter title

---
 youtube_dl/extractor/parler.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index b22de186ea4..efa8136d2d9 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -25,7 +25,8 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'df79fdba-07cc-48fe-b085-3293897520d7',
                 'ext': 'mp4',
-                'title': 'Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.',
+                'title': '@TulsiGabbard-720',
+                'description': 'Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.',
                 'timestamp': 1659744000,
                 'upload_date': '20220806',
                 'uploader': 'Tulsi Gabbard',
@@ -38,7 +39,8 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'a7406eb4-91e5-4793-b5e3-ade57a24e287',
                 'ext': 'mp4',
-                'title': 'This man should run for office',
+                'title': '@BennyJohnson-360',
+                'description': 'This man should run for office',
                 'timestamp': 1659657600,
                 'upload_date': '20220805',
                 'uploader': 'Benny Johnson',
@@ -61,17 +63,23 @@ def _real_extract(self, url):
         data = status['data'][0]['primary']
 
         # Pull out metadata
-        title = clean_html(data.get('full_body')) or self._generic_title(url)
+        description = strip_or_none(clean_html(data.get('full_body')))
         timestamp = unified_timestamp(data.get('date_created'))
         uploader = strip_or_none(data.get('name'))
         uploader_id = strip_or_none(data.get('username'))
         uploader_url = ('https://parler.com/' + uploader_id) if uploader_id else None
 
+        # Keep the file name short so it doesn't exceed filesystem limits
+        title = self._generic_title(url)
+        if uploader_id:
+            title = '@%s-%s' % (uploader_id, title)
+
         # Return the result
         return {
             'id': video_id,
             'url': url,
             'title': title,
+            'description': description,
             'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,

From c8686e7ccb09b4d16c01e74bcfbc7b16c5758cdf Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Tue, 9 Aug 2022 03:44:59 -0700
Subject: [PATCH 356/394] No @

---
 youtube_dl/extractor/parler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index efa8136d2d9..96a955b4d78 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -25,7 +25,7 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'df79fdba-07cc-48fe-b085-3293897520d7',
                 'ext': 'mp4',
-                'title': '@TulsiGabbard-720',
+                'title': 'TulsiGabbard-720',
                 'description': 'Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.',
                 'timestamp': 1659744000,
                 'upload_date': '20220806',
@@ -39,7 +39,7 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'a7406eb4-91e5-4793-b5e3-ade57a24e287',
                 'ext': 'mp4',
-                'title': '@BennyJohnson-360',
+                'title': 'BennyJohnson-360',
                 'description': 'This man should run for office',
                 'timestamp': 1659657600,
                 'upload_date': '20220805',
@@ -72,7 +72,7 @@ def _real_extract(self, url):
         # Keep the file name short so it doesn't exceed filesystem limits
         title = self._generic_title(url)
         if uploader_id:
-            title = '@%s-%s' % (uploader_id, title)
+            title = '%s-%s' % (uploader_id, title)
 
         # Return the result
         return {

From 789a9293496455695d64949384279ae140cec736 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Tue, 9 Aug 2022 03:47:50 -0700
Subject: [PATCH 357/394] Trim

---
 youtube_dl/extractor/truth.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index b4827377980..c1d83b2ee03 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -22,7 +22,7 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108779000807761862',
                 'ext': 'qt',
-                'title': '0d8691160c73d663',
+                'title': 'realDonaldTrump-0d8691160c73d663',
                 'timestamp': 1659835827,
                 'upload_date': '20220807',
                 'uploader': 'Donald J. Trump',
@@ -35,7 +35,8 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108618228543962049',
                 'ext': 'mp4',
-                'title': """RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  \n“Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe""",
+                'title': 'ProjectVeritasAction-6e24b75a4604b594',
+                'description': """RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  \n“Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe""",
                 'timestamp': 1657382637,
                 'upload_date': '20220709',
                 'uploader': 'Project Veritas Action',
@@ -56,7 +57,7 @@ def _real_extract(self, url):
         url = status['media_attachments'][0]['url']
 
         # Pull out metadata
-        title = strip_or_none(clean_html(status.get('content'))) or self._generic_title(url)
+        description = strip_or_none(clean_html(status.get('content')))
         timestamp = unified_timestamp(status.get('created_at'))
         account = status.get('account') or {}
         uploader = strip_or_none(account.get('display_name'))
@@ -66,11 +67,17 @@ def _real_extract(self, url):
         like_count = int_or_none(status.get('favourites_count'))
         comment_count = int_or_none(status.get('replies_count'))
 
+        # Keep the file name short so it doesn't exceed filesystem limits
+        title = self._generic_title(url)
+        if uploader_id:
+            title = '%s-%s' % (uploader_id, title)
+
         # Return the stuff
         return {
             'id': video_id,
             'url': url,
             'title': title,
+            'description': description,
             'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,

From f2a599cf77a1c5d70c44e9e8d56e59ca61805112 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 19:34:34 +0100
Subject: [PATCH 358/394] [test] Implement string "lambda x: condition(x)" as
 an expected value

Semantics equivalent to `assert condition(got)`
---
 test/helper.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index e62aab11e77..c6a2f06670d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,6 +128,12 @@ def expect_value(self, got, expected, field):
         self.assertTrue(
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
+    elif isinstance(expected, compat_str) and re.match(r'^lambda \w+:', expected):
+        fn = eval(expected)
+        suite = expected.split(':', 1)[1].strip()
+        self.assertTrue(
+            fn(got),
+            'Expected field %s to meet condition %s, but value %r failed ' % (field, suite, got))
     elif isinstance(expected, type):
         self.assertTrue(
             isinstance(got, expected),
@@ -137,7 +143,7 @@ def expect_value(self, got, expected, field):
     elif isinstance(expected, list) and isinstance(got, list):
         self.assertEqual(
             len(expected), len(got),
-            'Expect a list of length %d, but got a list of length %d for field %s' % (
+            'Expected a list of length %d, but got a list of length %d for field %s' % (
                 len(expected), len(got), field))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)

From 1d15ed47c2b3539c8b3b2336f23b54f2b1766cbf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 19:37:58 +0100
Subject: [PATCH 359/394] [ZDF] Overhaul ZDF extractors * pull some yt-dlp
 changes into ZDFBaseIE._extract_format() * add test cases from yt-dlp to
 ZDFIE * fix crash in ZDFIE._extract_mobile() when object had no `formitaeten`
 * improve title extraction in ZDFChannelIE (remove trailing station ident) *
 avoid extracting non-video playlist items (fixes #31149)

---
 youtube_dl/extractor/zdf.py | 175 ++++++++++++++++++++----------------
 1 file changed, 100 insertions(+), 75 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 3d39bb33aec..fcc63ef52ca 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -8,13 +8,14 @@
 from ..utils import (
     determine_ext,
     ExtractorError,
+    extract_attributes,
     float_or_none,
     int_or_none,
     merge_dicts,
     NO_DEFAULT,
-    orderedSet,
     parse_codecs,
     qualities,
+    str_or_none,
     try_get,
     unified_timestamp,
     update_url_query,
@@ -57,28 +58,39 @@ def _extract_format(self, video_id, formats, format_urls, meta):
         format_urls.add(format_url)
         mime_type = meta.get('mimeType')
         ext = determine_ext(format_url)
+
+        join_nonempty = lambda s, l: s.join(filter(None, l))
+        meta_map = lambda t: map(lambda x: str_or_none(meta.get(x)), t)
+
         if mime_type == 'application/x-mpegURL' or ext == 'm3u8':
-            formats.extend(self._extract_m3u8_formats(
+            new_formats = self._extract_m3u8_formats(
                 format_url, video_id, 'mp4', m3u8_id='hls',
-                entry_protocol='m3u8_native', fatal=False))
+                entry_protocol='m3u8_native', fatal=False)
         elif mime_type == 'application/f4m+xml' or ext == 'f4m':
-            formats.extend(self._extract_f4m_formats(
-                update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False))
+            new_formats = self._extract_f4m_formats(
+                update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False)
         else:
             f = parse_codecs(meta.get('mimeCodec'))
+            if not f:
+                data = meta.get('type', '').split('_')
+                if try_get(data, lambda x: x[2]) == ext:
+                    f = dict(zip(('vcodec', 'acodec'), data[1]))
+
             format_id = ['http']
-            for p in (meta.get('type'), meta.get('quality')):
-                if p and isinstance(p, compat_str):
-                    format_id.append(p)
+            format_id.extend(join_nonempty('-', meta_map(('type', 'quality'))))
             f.update({
                 'url': format_url,
                 'format_id': '-'.join(format_id),
-                'format_note': meta.get('quality'),
-                'language': meta.get('language'),
-                'quality': qualities(self._QUALITIES)(meta.get('quality')),
-                'preference': -10,
+                'tbr': int_or_none(self._search_regex(r'_(\d+)k_', format_url, 'tbr', default=None))
             })
-            formats.append(f)
+            new_formats = [f]
+
+        formats.extend(merge_dicts(f, {
+            'format_note': join_nonempty(',', meta_map(('quality', 'class'))),
+            'language': meta.get('language'),
+            'language_preference': 10 if meta.get('class') == 'main' else -10 if meta.get('class') == 'ad' else -1,
+            'quality': qualities(self._QUALITIES)(meta.get('quality')),
+        }) for f in new_formats)
 
     def _extract_ptmd(self, ptmd_url, video_id, api_token, referrer):
         ptmd = self._call_api(
@@ -107,6 +119,7 @@ def _extract_ptmd(self, ptmd_url, video_id, api_token, referrer):
                                 'type': f.get('type'),
                                 'mimeType': f.get('mimeType'),
                                 'quality': quality.get('quality'),
+                                'class': track.get('class'),
                                 'language': track.get('language'),
                             })
         self._sort_formats(formats)
@@ -171,6 +184,20 @@ class ZDFIE(ZDFBaseIE):
             'duration': 2615,
             'timestamp': 1465021200,
             'upload_date': '20160604',
+            'thumbnail': 'https://www.zdf.de/assets/mauve-im-labor-100~768x432?cb=1464909117806',
+        },
+    }, {
+        'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
+        'md5': '1b93bdec7d02fc0b703c5e7687461628',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'video_funk_1770473',
+            'duration': 1278,
+            'description': 'Die Neue an der Schule verdreht Ismail den Kopf.',
+            'title': 'Alles ist verzaubert',
+            'timestamp': 1635520560,
+            'upload_date': '20211029',
+            'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-100~1920x1080?cb=1636466431799',
         },
     }, {
         # Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche
@@ -204,6 +231,19 @@ class ZDFIE(ZDFBaseIE):
             'timestamp': 1641355200,
             'upload_date': '20220105',
         },
+        'skip': 'No longer available "Diese Seite wurde leider nicht gefunden"'
+    }, {
+        'url': 'https://www.zdf.de/serien/soko-stuttgart/das-geld-anderer-leute-100.html',
+        'info_dict': {
+            'id': '191205_1800_sendung_sok8',
+            'ext': 'mp4',
+            'title': 'Das Geld anderer Leute',
+            'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
+            'duration': 2581.0,
+            'timestamp': 1654790700,
+            'upload_date': '20220609',
+            'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350',
+        },
     }]
 
     def _extract_entry(self, url, player, content, video_id):
@@ -265,15 +305,16 @@ def _extract_mobile(self, video_id):
             'https://zdf-cdn.live.cellular.de/mediathekV2/document/%s' % video_id,
             video_id)
 
-        document = video['document']
-
-        title = document['titel']
-        content_id = document['basename']
-
         formats = []
-        format_urls = set()
-        for f in document['formitaeten']:
-            self._extract_format(content_id, formats, format_urls, f)
+        formitaeten = try_get(video, lambda x: x['document']['formitaeten'], list)
+        document = formitaeten and video['document']
+        if formitaeten:
+            title = document['titel']
+            content_id = document['basename']
+
+            format_urls = set()
+            for f in formitaeten or []:
+                self._extract_format(content_id, formats, format_urls, f)
         self._sort_formats(formats)
 
         thumbnails = []
@@ -320,9 +361,9 @@ class ZDFChannelIE(ZDFBaseIE):
         'url': 'https://www.zdf.de/sport/das-aktuelle-sportstudio',
         'info_dict': {
             'id': 'das-aktuelle-sportstudio',
-            'title': 'das aktuelle sportstudio | ZDF',
+            'title': 'das aktuelle sportstudio',
         },
-        'playlist_mincount': 23,
+        'playlist_mincount': 18,
     }, {
         'url': 'https://www.zdf.de/dokumentation/planet-e',
         'info_dict': {
@@ -330,6 +371,14 @@ class ZDFChannelIE(ZDFBaseIE):
             'title': 'planet e.',
         },
         'playlist_mincount': 50,
+    }, {
+        'url': 'https://www.zdf.de/gesellschaft/aktenzeichen-xy-ungeloest',
+        'info_dict': {
+            'id': 'aktenzeichen-xy-ungeloest',
+            'title': 'Aktenzeichen XY... ungelöst',
+            'entries': "lambda x: not any('xy580-fall1-kindermoerder-gesucht-100' in e['url'] for e in x)",
+        },
+        'playlist_mincount': 2,
     }, {
         'url': 'https://www.zdf.de/filme/taunuskrimi/',
         'only_matching': True,
@@ -339,60 +388,36 @@ class ZDFChannelIE(ZDFBaseIE):
     def suitable(cls, url):
         return False if ZDFIE.suitable(url) else super(ZDFChannelIE, cls).suitable(url)
 
+    def _og_search_title(self, webpage, fatal=False):
+        title = super(ZDFChannelIE, self)._og_search_title(webpage, fatal=fatal)
+        return re.split(r'\s+[-|]\s+ZDF(?:mediathek)?$', title or '')[0] or None
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
         webpage = self._download_webpage(url, channel_id)
 
-        entries = [
-            self.url_result(item_url, ie=ZDFIE.ie_key())
-            for item_url in orderedSet(re.findall(
-                r'data-plusbar-url=["\'](http.+?\.html)', webpage))]
-
-        return self.playlist_result(
-            entries, channel_id, self._og_search_title(webpage, fatal=False))
-
-        r"""
-        player = self._extract_player(webpage, channel_id)
-
-        channel_id = self._search_regex(
-            r'docId\s*:\s*(["\'])(?P<id>(?!\1).+?)\1', webpage,
-            'channel id', group='id')
-
-        channel = self._call_api(
-            'https://api.zdf.de/content/documents/%s.json' % channel_id,
-            player, url, channel_id)
-
-        items = []
-        for module in channel['module']:
-            for teaser in try_get(module, lambda x: x['teaser'], list) or []:
-                t = try_get(
-                    teaser, lambda x: x['http://zdf.de/rels/target'], dict)
-                if not t:
-                    continue
-                items.extend(try_get(
-                    t,
-                    lambda x: x['resultsWithVideo']['http://zdf.de/rels/search/results'],
-                    list) or [])
-            items.extend(try_get(
-                module,
-                lambda x: x['filterRef']['resultsWithVideo']['http://zdf.de/rels/search/results'],
-                list) or [])
-
-        entries = []
-        entry_urls = set()
-        for item in items:
-            t = try_get(item, lambda x: x['http://zdf.de/rels/target'], dict)
-            if not t:
-                continue
-            sharing_url = t.get('http://zdf.de/rels/sharing-url')
-            if not sharing_url or not isinstance(sharing_url, compat_str):
-                continue
-            if sharing_url in entry_urls:
-                continue
-            entry_urls.add(sharing_url)
-            entries.append(self.url_result(
-                sharing_url, ie=ZDFIE.ie_key(), video_id=t.get('id')))
-
-        return self.playlist_result(entries, channel_id, channel.get('title'))
-        """
+        matches = re.finditer(
+            r'''<div\b[^>]*?\sdata-plusbar-id\s*=\s*(["'])(?P<p_id>[\w-]+)\1[^>]*?\sdata-plusbar-url=\1(?P<url>%s)\1''' % ZDFIE._VALID_URL,
+            webpage)
+
+        if self._downloader.params.get('noplaylist', False):
+            entry = next(
+                (self.url_result(m.group('url'), ie=ZDFIE.ie_key()) for m in matches),
+                None)
+            self.to_screen('Downloading just the main video because of --no-playlist')
+            if entry:
+                return entry
+        else:
+            self.to_screen('Downloading playlist %s - add --no-playlist to download just the main video' % (channel_id, ))
+
+        def check_video(m):
+            v_ref = self._search_regex(
+                r'''(<a\b[^>]*?\shref\s*=[^>]+?\sdata-target-id\s*=\s*(["'])%s\2[^>]*>)''' % (m.group('p_id'), ),
+                webpage, 'check id', default='')
+            v_ref = extract_attributes(v_ref)
+            return v_ref.get('data-target-video-type') != 'novideo'
+
+        return self.playlist_from_matches(
+            (m.group('url') for m in matches if check_video(m)),
+            channel_id, self._og_search_title(webpage, fatal=False))

From 27527397e47fc9ea6ca4984fed5721c044222a8a Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Wed, 10 Aug 2022 04:54:11 -0700
Subject: [PATCH 360/394] Clean it up

---
 youtube_dl/extractor/truth.py | 63 +++++++++++++++++++----------------
 1 file changed, 34 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index c1d83b2ee03..5cbfd8eecb8 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -1,6 +1,3 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 from .common import InfoExtractor
 
 from ..utils import (
@@ -14,7 +11,7 @@
 class TruthIE(InfoExtractor):
     """Extract videos from posts on Donald Trump's truthsocial.com."""
 
-    _VALID_URL = r'https://truthsocial\.com/@[^/]+/posts/(?P<id>[0-9]+)'
+    _VALID_URL = r'https://truthsocial\.com/@[^/]+/posts/(?P<id>[\d]+)'
     _TESTS = [
         {
             'url': 'https://truthsocial.com/@realDonaldTrump/posts/108779000807761862',
@@ -22,11 +19,15 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108779000807761862',
                 'ext': 'qt',
-                'title': 'realDonaldTrump-0d8691160c73d663',
+                'title': 'Donald J. Trump - 0d8691160c73d663',
                 'timestamp': 1659835827,
                 'upload_date': '20220807',
                 'uploader': 'Donald J. Trump',
                 'uploader_id': 'realDonaldTrump',
+                'uploader_url': 'https://truthsocial.com/@realDonaldTrump',
+                'repost_count': int,
+                'comment_count': int,
+                'like_count': int,
             },
         },
         {
@@ -35,54 +36,58 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108618228543962049',
                 'ext': 'mp4',
-                'title': 'ProjectVeritasAction-6e24b75a4604b594',
-                'description': """RETRACTO #368: Utah NPR Affiliate RETRACTS False Claim Live On Air Following Veritas' Reporting on Curtis Campaign  \n“Nothing I ever do will suffice for these people. They are engaged in conspiracy theories. They are doing precisely the thing they project that I do. Which is they don’t believe in facts, they don’t believe in logic, and they don’t believe in rationality.” - James O’Keefe""",
+                'title': 'md5:48813a16498d21b07edf24e1af621e83',
+                'description': 'md5:e070ba6bcf6165957e26a7a94ef6d975',
                 'timestamp': 1657382637,
                 'upload_date': '20220709',
                 'uploader': 'Project Veritas Action',
                 'uploader_id': 'ProjectVeritasAction',
+                'uploader_url': 'https://truthsocial.com/@ProjectVeritasAction',
+                'repost_count': int,
+                'comment_count': int,
+                'like_count': int,
             },
         },
     ]
     _GEO_COUNTRIES = ['US']  # The site is only available in the US
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
         # Get data from API
-        api_url = 'https://truthsocial.com/api/v1/statuses/' + video_id
-        status = self._download_json(api_url, video_id)
+        video_id = self._match_id(url)
+        status = self._download_json(
+            'https://truthsocial.com/api/v1/statuses/' + video_id,
+            video_id
+        )
 
         # Pull out video
         url = status['media_attachments'][0]['url']
 
-        # Pull out metadata
-        description = strip_or_none(clean_html(status.get('content')))
-        timestamp = unified_timestamp(status.get('created_at'))
+        # Return the stuff
         account = status.get('account') or {}
         uploader = strip_or_none(account.get('display_name'))
         uploader_id = strip_or_none(account.get('username'))
-        uploader_url = ('https://truthsocial.com/@' + uploader_id) if uploader_id else None
-        repost_count = int_or_none(status.get('reblogs_count'))
-        like_count = int_or_none(status.get('favourites_count'))
-        comment_count = int_or_none(status.get('replies_count'))
+        post = strip_or_none(clean_html(status.get('content')))
 
-        # Keep the file name short so it doesn't exceed filesystem limits
-        title = self._generic_title(url)
-        if uploader_id:
-            title = '%s-%s' % (uploader_id, title)
+        # Set the title, handling case where its too long or empty
+        if len(post) > 40:
+            title = post[:35] + "[...]"
+        elif len(post) == 0:
+            title = self._generic_title(url)
+        else:
+            title = post
+        if uploader:
+            title = '%s - %s' % (uploader, title)
 
-        # Return the stuff
         return {
             'id': video_id,
             'url': url,
             'title': title,
-            'description': description,
-            'timestamp': timestamp,
+            'description': post,
+            'timestamp': unified_timestamp(status.get('created_at')),
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'uploader_url': uploader_url,
-            'repost_count': repost_count,
-            'like_count': like_count,
-            'comment_count': comment_count,
+            'uploader_url': ('https://truthsocial.com/@' + uploader_id) if uploader_id else None,
+            'repost_count': int_or_none(status.get('reblogs_count')),
+            'like_count': int_or_none(status.get('favourites_count')),
+            'comment_count': int_or_none(status.get('replies_count')),
         }

From 07e143b3268756154deb58f4aaa6e642484c18d0 Mon Sep 17 00:00:00 2001
From: Ben Welsh <b@palewi.re>
Date: Wed, 10 Aug 2022 05:04:39 -0700
Subject: [PATCH 361/394] Update truth.py

---
 youtube_dl/extractor/truth.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index 5cbfd8eecb8..1d8a671a226 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 
 from ..utils import (

From 6bfc29ec02c687fa5f5f9b0d8f194de338e28650 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 14 Aug 2022 17:44:02 +0100
Subject: [PATCH 362/394] [jsinterp] Overhaul JSInterp to handle new YT players
 4c3f79c5, 324f67b9 * back-port from yt-dlp
 8f53dc44a0cc1c2d98c35740b9293462c080f5d0, thanks pukkandan * also support
 void, improve <</>> precedence, improve expressions in comma-list * add more
 tests

---
 test/test_jsinterp.py          |  49 ++-
 test/test_utils.py             |   3 +
 test/test_youtube_signature.py |  13 +
 youtube_dl/compat.py           |  54 ++-
 youtube_dl/jsinterp.py         | 589 ++++++++++++++++++++-------------
 youtube_dl/utils.py            |  47 ++-
 6 files changed, 504 insertions(+), 251 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index acdabffb1f8..c6c93174353 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ def test_basic(self):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('function x3(){42}')
+        self.assertEqual(jsi.call_function('x3'), None)
+
         jsi = JSInterpreter('var x5 = function(){return 42;}')
         self.assertEqual(jsi.call_function('x5'), 42)
 
@@ -51,8 +54,11 @@ def test_operators(self):
         jsi = JSInterpreter('function f(){return 11 >> 2;}')
         self.assertEqual(jsi.call_function('f'), 2)
 
+        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
+        self.assertEqual(jsi.call_function('f'), 5)
+
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
 
     def test_parens(self):
@@ -62,6 +68,10 @@ def test_parens(self):
         jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
         self.assertEqual(jsi.call_function('f'), 9)
 
+    def test_quotes(self):
+        jsi = JSInterpreter(r'function f(){return "a\"\\("}')
+        self.assertEqual(jsi.call_function('f'), r'a"\(')
+
     def test_assignments(self):
         jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
         self.assertEqual(jsi.call_function('f'), 31)
@@ -104,18 +114,29 @@ def test_precedence(self):
         }''')
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
+    def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 86000)
+        jsi = JSInterpreter('''
+        function x(dt) { return new Date(dt) - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+
     def test_call(self):
         jsi = JSInterpreter('''
         function x() { return 2; }
-        function y(a) { return x() + a; }
+        function y(a) { return x() + (a?a:0); }
         function z() { return y(3); }
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
+        self.assertEqual(jsi.call_function('y'), 2)
 
     def test_for_loop(self):
         # function x() { a=0; for (i=0; i-10; i++) {a++} a }
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
@@ -156,19 +177,19 @@ def test_try(self):
 
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_for_loop_break(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_literal_list(self):
         jsi = JSInterpreter('''
-        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
         ''')
         self.assertEqual(jsi.call_function('x'), [5, 6, 7])
 
@@ -177,6 +198,22 @@ def test_comma(self):
         function x() { a=5; a -= 1, a+=3; return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
+        jsi = JSInterpreter('''
+        function x() { a=5; return (a -= 1, a+=3, a); }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
+    def test_void(self):
+        jsi = JSInterpreter('''
+        function x() { return void 42; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+    def test_return_function(self):
+        jsi = JSInterpreter('''
+        function x() { return [1, function(){return 1}][1] }
+        ''')
+        self.assertEqual(jsi.call_function('x')([]), 1)
 
 
 if __name__ == '__main__':
diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e1e..f1a748ddee8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,9 @@ def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
         self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
         self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
+        self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1)
+        self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86)
+        self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index fc5e9828e2c..6e955e0f019 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -90,12 +90,25 @@
         'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
         'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
     ),
+    (
+        'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
+        'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
+    ),
+    (
+        'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
+        'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
+    ),
+    (
+        'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
+        'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
+    ),
 ]
 
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
         PLAYER_URLS = (
+            ('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9f5f85daeaa..6d2c31a61c5 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2985,7 +2985,6 @@ def unpack(self, string):
     except ImportError:
         compat_filter = filter
 
-
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -2995,6 +2994,57 @@ def unpack(self, string):
         compat_zip = zip
 
 
+# method renamed between Py2/3
+try:
+    from itertools import zip_longest as compat_itertools_zip_longest
+except ImportError:
+    from itertools import izip_longest as compat_itertools_zip_longest
+
+
+# new class in collections
+try:
+    from collections import ChainMap as compat_collections_chain_map
+except ImportError:
+    # Py < 3.3
+    class compat_collections_chain_map(compat_collections_abc.MutableMapping):
+
+        maps = [{}]
+
+        def __init__(self, *maps):
+            self.maps = list(maps) or [{}]
+
+        def __getitem__(self, k):
+            for m in self.maps:
+                if k in m:
+                    return m[k]
+            raise KeyError(k)
+
+        def __setitem__(self, k, v):
+            self.maps[0].__setitem__(k, v)
+            return
+
+        def __delitem__(self, k):
+            if k in self.maps[0]:
+                del self.maps[0][k]
+                return
+            raise KeyError(k)
+
+        def __iter__(self):
+            return itertools.chain(*reversed(self.maps))
+
+        def __len__(self):
+            return len(iter(self))
+
+        def new_child(self, m=None, **kwargs):
+            m = m or {}
+            m.update(kwargs)
+            return compat_collections_chain_map(m, *self.maps)
+
+        @property
+        def parents(self):
+            return compat_collections_chain_map(*(self.maps[1:]))
+
+
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
         if isinstance(s, compat_str):
@@ -3031,6 +3081,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_basestring',
     'compat_chr',
     'compat_collections_abc',
+    'compat_collections_chain_map',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
@@ -3051,6 +3102,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_input',
     'compat_integer_types',
     'compat_itertools_count',
+    'compat_itertools_zip_longest',
     'compat_kwargs',
     'compat_map',
     'compat_numeric_types',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8eaa911cdab..c60a9b3c234 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,42 +1,87 @@
 from __future__ import unicode_literals
 
+import itertools
 import json
+import math
 import operator
 import re
 
 from .utils import (
+    NO_DEFAULT,
     ExtractorError,
+    js_to_json,
     remove_quotes,
+    unified_timestamp,
 )
 from .compat import (
-    compat_collections_abc,
+    compat_collections_chain_map as ChainMap,
+    compat_itertools_zip_longest as zip_longest,
     compat_str,
 )
-MutableMapping = compat_collections_abc.MutableMapping
 
+_NAME_RE = r'[a-zA-Z_$][\w$]*'
 
-class Nonlocal:
-    pass
-
+# (op, definition) in order of binding priority, tightest first
+# avoid dict to maintain order
+# definition None => Defined in JSInterpreter._operator
+_DOT_OPERATORS = (
+    ('.', None),
+    # TODO: ('?.', None),
+)
 
-_OPERATORS = [
+_OPERATORS = (
     ('|', operator.or_),
     ('^', operator.xor),
     ('&', operator.and_),
     ('>>', operator.rshift),
     ('<<', operator.lshift),
-    ('-', operator.sub),
     ('+', operator.add),
-    ('%', operator.mod),
-    ('/', operator.truediv),
+    ('-', operator.sub),
     ('*', operator.mul),
-]
-_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
+    ('/', operator.truediv),
+    ('%', operator.mod),
+)
 
-_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+_COMP_OPERATORS = (
+    ('===', operator.is_),
+    ('==', operator.eq),
+    ('!==', operator.is_not),
+    ('!=', operator.ne),
+    ('<=', operator.le),
+    ('>=', operator.ge),
+    ('<', operator.lt),
+    ('>', operator.gt),
+)
+
+_LOG_OPERATORS = (
+    ('&', operator.and_),
+    ('|', operator.or_),
+    ('^', operator.xor),
+)
+
+_SC_OPERATORS = (
+    ('?', None),
+    ('||', None),
+    ('&&', None),
+    # TODO: ('??', None),
+)
+
+_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+_QUOTES = '\'"'
+
+
+def _ternary(cndn, if_true=True, if_false=False):
+    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
+    if cndn in (False, None, 0, ''):
+        return if_false
+    try:
+        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
+            return if_false
+    except TypeError:
+        pass
+    return if_true
 
 
 class JS_Break(ExtractorError):
@@ -49,70 +94,77 @@ def __init__(self):
         ExtractorError.__init__(self, 'Invalid continue')
 
 
-class LocalNameSpace(MutableMapping):
-    def __init__(self, *stack):
-        self.stack = tuple(stack)
-
-    def __getitem__(self, key):
-        for scope in self.stack:
-            if key in scope:
-                return scope[key]
-        raise KeyError(key)
-
+class LocalNameSpace(ChainMap):
     def __setitem__(self, key, value):
-        for scope in self.stack:
+        for scope in self.maps:
             if key in scope:
                 scope[key] = value
-                break
-        else:
-            self.stack[0][key] = value
-        return value
+                return
+        self.maps[0][key] = value
 
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
-    def __iter__(self):
-        for scope in self.stack:
-            for scope_item in iter(scope):
-                yield scope_item
-
-    def __len__(self, key):
-        return len(iter(self))
-
     def __repr__(self):
-        return 'LocalNameSpace%s' % (self.stack, )
+        return 'LocalNameSpace%s' % (self.maps, )
 
 
 class JSInterpreter(object):
+    __named_object_counter = 0
+
     def __init__(self, code, objects=None):
-        if objects is None:
-            objects = {}
-        self.code = code
-        self._functions = {}
-        self._objects = objects
-        self.__named_object_counter = 0
+        self.code, self._functions = code, {}
+        self._objects = {} if objects is None else objects
+
+    class Exception(ExtractorError):
+        def __init__(self, msg, *args, **kwargs):
+            expr = kwargs.pop('expr', None)
+            if expr is not None:
+                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
+            super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
-        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
         namespace[name] = obj
         return name
 
     @staticmethod
-    def _separate(expr, delim=',', max_split=None):
+    def _separate(expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
+        start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
+        in_quote, escaping = None, False
         for idx, char in enumerate(expr):
-            if char in _MATCHING_PARENS:
-                counters[_MATCHING_PARENS[char]] += 1
-            elif char in counters:
-                counters[char] -= 1
-            if char != delim[pos] or any(counters.values()):
-                pos = 0
+            if not in_quote:
+                if char in _MATCHING_PARENS:
+                    counters[_MATCHING_PARENS[char]] += 1
+                elif char in counters:
+                    counters[char] -= 1
+            if not escaping:
+                if char in _QUOTES and in_quote in (char, None):
+                    in_quote = None if in_quote else char
+                else:
+                    escaping = in_quote and char == '\\'
+            else:
+                escaping = False
+
+            if char != delim[pos] or any(counters.values()) or in_quote:
+                pos = skipping = 0
                 continue
-            elif pos != delim_len:
+            elif skipping > 0:
+                skipping -= 1
+                continue
+            elif pos == 0 and skip_delims:
+                here = expr[idx:]
+                for s in skip_delims if isinstance(skip_delims, (list, tuple)) else [skip_delims]:
+                    if here.startswith(s) and s:
+                        skipping = len(s) - 1
+                        break
+                if skipping > 0:
+                    continue
+            if pos < delim_len:
                 pos += 1
                 continue
             yield expr[start: idx - delim_len]
@@ -122,61 +174,108 @@ def _separate(expr, delim=',', max_split=None):
                 break
         yield expr[start:]
 
-    @staticmethod
-    def _separate_at_paren(expr, delim):
-        separated = list(JSInterpreter._separate(expr, delim, 1))
+    @classmethod
+    def _separate_at_paren(cls, expr, delim):
+        separated = list(cls._separate(expr, delim, 1))
+
         if len(separated) < 2:
-            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
+    @staticmethod
+    def _all_operators():
+        return itertools.chain(
+            _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
+
+    def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
+        if op in ('||', '&&'):
+            if (op == '&&') ^ _ternary(left_val):
+                return left_val  # short circuiting
+        elif op == '?':
+            right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
+
+        right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
+        opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
+        if not opfunc:
+            return right_val
+
+        try:
+            return opfunc(left_val, right_val)
+        except Exception as e:
+            raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
+
+    def _index(self, obj, idx):
+        if idx == 'length':
+            return len(obj)
+        try:
+            return obj[int(idx)] if isinstance(obj, list) else obj[idx]
+        except Exception as e:
+            raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
+
+    def _dump(self, obj, namespace):
+        try:
+            return json.dumps(obj)
+        except TypeError:
+            return self._named_object(namespace, obj)
+
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
-            raise ExtractorError('Recursion limit reached')
+            raise self.Exception('Recursion limit reached')
+        allow_recursion -= 1
 
-        sub_statements = list(self._separate(stmt, ';'))
-        stmt = (sub_statements or ['']).pop()
+        should_return = False
+        sub_statements = list(self._separate(stmt, ';')) or ['']
+        expr = stmt = sub_statements.pop().strip()
         for sub_stmt in sub_statements:
-            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
-            if should_abort:
-                return ret
+            ret, should_return = self.interpret_statement(sub_stmt, local_vars, allow_recursion)
+            if should_return:
+                return ret, should_return
 
-        should_abort = False
-        stmt = stmt.lstrip()
-        stmt_m = re.match(r'var\s', stmt)
-        if stmt_m:
-            expr = stmt[len(stmt_m.group(0)):]
-        else:
-            return_m = re.match(r'return(?:\s+|$)', stmt)
-            if return_m:
-                expr = stmt[len(return_m.group(0)):]
-                should_abort = True
-            else:
-                # Try interpreting it as an expression
-                expr = stmt
+        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
+        if m:
+            expr = stmt[len(m.group(0)):].strip()
+            should_return = not m.group('var')
+        if not expr:
+            return None, should_return
 
-        v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return v, should_abort
+        if expr[0] in _QUOTES:
+            inner, outer = self._separate(expr, expr[0], 1)
+            inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
+            if not outer:
+                return inner, should_return
+            expr = self._named_object(local_vars, inner) + outer
+
+        if expr.startswith('new '):
+            obj = expr[4:]
+            if obj.startswith('Date('):
+                left, right = self._separate_at_paren(obj[4:], ')')
+                left = self.interpret_expression(left, local_vars, allow_recursion)
+                expr = unified_timestamp(left, False)
+                if not expr:
+                    raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
+                expr = self._dump(int(expr * 1000), local_vars) + right
+            else:
+                raise self.Exception('Unsupported object {obj}'.format(**locals()), expr=expr)
 
-    def interpret_expression(self, expr, local_vars, allow_recursion):
-        expr = expr.strip()
-        if expr == '':  # Empty expression
-            return None
+        if expr.startswith('void '):
+            left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
+            return None, should_return
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
-            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
-                return inner
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
             inner, outer = self._separate_at_paren(expr, ')')
-            inner = self.interpret_expression(inner, local_vars, allow_recursion)
-            if not outer:
-                return inner
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
+            if not outer or should_abort:
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('['):
             inner, outer = self._separate_at_paren(expr, ']')
@@ -185,57 +284,53 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'try\s*', expr)
-        if m:
+        m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('try'):
             if expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
-            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
             if should_abort:
-                return ret
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                return ret, True
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
-        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
-        md = m.groupdict() if m else {}
-        if md.get('catch'):
+        elif md.get('catch'):
             # We ignore the catch block
             _, expr = self._separate_at_paren(expr, '}')
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('for'):
-            def raise_constructor_error(c):
-                raise ExtractorError(
-                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
-
             constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             if remaining.startswith('{'):
                 body, expr = self._separate_at_paren(remaining, '}')
             else:
-                m = re.match(r'switch\s*\(', remaining)  # FIXME
-                if m:
-                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if switch_m:
+                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
                     body, expr = self._separate_at_paren(remaining, '}')
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
                     body, expr = remaining, ''
             start, cndn, increment = self._separate(constructor, ';')
-            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
-                raise_constructor_error(constructor)
+            self.interpret_expression(start, local_vars, allow_recursion)
             while True:
-                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                     break
                 try:
-                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
                     if should_abort:
-                        return ret
+                        return ret, True
                 except JS_Break:
                     break
                 except JS_Continue:
                     pass
-                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
-                    raise_constructor_error(constructor)
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                self.interpret_expression(increment, local_vars, allow_recursion)
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('switch'):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
@@ -245,7 +340,7 @@ def raise_constructor_error(c):
             for default in (False, True):
                 matched = False
                 for item in items:
-                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    case, stmt = (i.strip() for i in self._separate(item, ':', 1))
                     if default:
                         matched = matched or case == 'default'
                     elif not matched:
@@ -254,24 +349,28 @@ def raise_constructor_error(c):
                     if not matched:
                         continue
                     try:
-                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
                 if matched:
                     break
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         # Comma separated statements
         sub_expressions = list(self._separate(expr))
-        expr = sub_expressions.pop().strip() if sub_expressions else ''
-        for sub_expr in sub_expressions:
-            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+        if len(sub_expressions) > 1:
+            for sub_expr in sub_expressions:
+                ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+            return ret, False
 
         for m in re.finditer(r'''(?x)
-                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
-                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+                (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
+                (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)'''.format(**globals()), expr):
             var = m.group('var1') or m.group('var2')
             start, end = m.span()
             sign = m.group('pre_sign') or m.group('post_sign')
@@ -279,85 +378,87 @@ def raise_constructor_error(c):
             local_vars[var] += 1 if sign[0] == '+' else -1
             if m.group('pre_sign'):
                 ret = local_vars[var]
-            expr = expr[:start] + json.dumps(ret) + expr[end:]
-
-        for op, opfunc in _ASSIGN_OPERATORS:
-            m = re.match(r'''(?x)
-                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
-                \s*%s
-                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
-            if not m:
-                continue
-            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
-
-            if m.groupdict().get('index'):
-                lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
-                if not isinstance(idx, int):
-                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
-                cur = lvar[idx]
-                val = opfunc(cur, right_val)
-                lvar[idx] = val
-                return val
-            else:
-                cur = local_vars.get(m.group('out'))
-                val = opfunc(cur, right_val)
-                local_vars[m.group('out')] = val
-                return val
+            expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
 
-        if expr.isdigit():
-            return int(expr)
-
-        if expr == 'break':
+        if not expr:
+            return None, should_return
+
+        m = re.match(r'''(?x)
+            (?P<assign>
+                (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
+                (?P<op>{_OPERATOR_RE})?
+                =(?P<expr>.*)$
+            )|(?P<return>
+                (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
+            )|(?P<indexing>
+                (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
+            )|(?P<attribute>
+                (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+            )|(?P<function>
+                (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
+            )'''.format(**globals()), expr)
+        md = m.groupdict() if m else {}
+        if md.get('assign'):
+            left_val = local_vars.get(m.group('out'))
+
+            if not m.group('index'):
+                local_vars[m.group('out')] = self._operator(
+                    m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
+                return local_vars[m.group('out')], should_return
+            elif left_val is None:
+                raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
+
+            idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+            if not isinstance(idx, (int, float)):
+                raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
+            idx = int(idx)
+            left_val[idx] = self._operator(
+                m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
+            return left_val[idx], should_return
+
+        elif expr.isdigit():
+            return int(expr), should_return
+
+        elif expr == 'break':
             raise JS_Break()
         elif expr == 'continue':
             raise JS_Continue()
 
-        var_m = re.match(
-            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
-            expr)
-        if var_m:
-            return local_vars[var_m.group('name')]
+        elif md.get('return'):
+            return local_vars[m.group('name')], should_return
 
         try:
-            return json.loads(expr)
+            ret = json.loads(js_to_json(expr))  # strict=True)
+            if not md.get('attribute'):
+                return ret, should_return
         except ValueError:
             pass
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
-        if m:
+        if md.get('indexing'):
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
-            return val[idx]
-
-        def raise_expr_error(where, op, exp):
-            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+            return self._index(val, idx), should_return
 
-        for op, opfunc in _OPERATORS:
-            separated = list(self._separate(expr, op))
+        for op, _ in self._all_operators():
+            # hackety: </> have higher priority than <</>>, but don't confuse them
+            skip_delim = (op + op) if op in ('<', '>') else None
+            separated = list(self._separate(expr, op, skip_delims=skip_delim))
             if len(separated) < 2:
                 continue
-            right_val = separated.pop()
-            left_val = op.join(separated)
-            left_val, should_abort = self.interpret_statement(
-                left_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('left-side', op, expr)
-            right_val, should_abort = self.interpret_statement(
-                right_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('right-side', op, expr)
-            return opfunc(left_val or 0, right_val)
 
-        m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
-            expr)
-        if m:
-            variable = m.group('var')
-            nl = Nonlocal()
+            right_expr = separated.pop()
+            while op == '-' and len(separated) > 1 and not separated[-1].strip():
+                right_expr = '-' + right_expr
+                separated.pop()
+            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
+            return self._operator(op, 0 if left_val is None else left_val,
+                                  right_expr, expr, local_vars, allow_recursion), should_return
 
-            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+        if md.get('attribute'):
+            variable = m.group('var')
+            member = m.group('member')
+            if not member:
+                member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
             if arg_str.startswith('('):
                 arg_str, remaining = self._separate_at_paren(arg_str, ')')
@@ -367,25 +468,24 @@ def raise_expr_error(where, op, exp):
             def assertion(cndn, msg):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
-                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+                    raise ExtractorError('{member} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
-                # nonlocal member
-                member = nl.member
-                if variable == 'String':
-                    obj = compat_str
-                elif variable in local_vars:
-                    obj = local_vars[variable]
-                else:
+                if (variable, member) == ('console', 'debug'):
+                    return
+                types = {
+                    'String': compat_str,
+                    'Math': float,
+                }
+                obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
+                if obj is NO_DEFAULT:
                     if variable not in self._objects:
                         self._objects[variable] = self.extract_object(variable)
                     obj = self._objects[variable]
 
+                # Member access
                 if arg_str is None:
-                    # Member access
-                    if member == 'length':
-                        return len(obj)
-                    return obj[member]
+                    return self._index(obj, member)
 
                 # Function call
                 argvals = [
@@ -396,12 +496,17 @@ def eval_method():
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
-                    raise ExtractorError('Unsupported string method %s' % (member, ))
+                    raise self.Exception('Unsupported string method ' + member, expr=expr)
+                elif obj == float:
+                    if member == 'pow':
+                        assertion(len(argvals) == 2, 'takes two arguments')
+                        return argvals[0] ** argvals[1]
+                    raise self.Exception('Unsupported Math method ' + member, expr=expr)
 
                 if member == 'split':
                     assertion(argvals, 'takes one or more arguments')
-                    assertion(argvals == [''], 'with arguments is not implemented')
-                    return list(obj)
+                    assertion(len(argvals) == 1, 'with limit argument is not implemented')
+                    return obj.split(argvals[0]) if argvals[0] else list(obj)
                 elif member == 'join':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(len(argvals) == 1, 'takes exactly one argument')
@@ -447,7 +552,7 @@ def eval_method():
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
                     f, this = (argvals + [''])[:2]
-                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                    return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
                 elif member == 'indexOf':
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
@@ -457,32 +562,35 @@ def eval_method():
                     except ValueError:
                         return -1
 
-                if isinstance(obj, list):
-                    member = int(member)
-                    nl.member = member
-                return obj[member](argvals)
+                idx = int(member) if isinstance(obj, list) else member
+                return obj[idx](argvals, allow_recursion=allow_recursion)
 
             if remaining:
-                return self.interpret_expression(
+                ret, should_abort = self.interpret_statement(
                     self._named_object(local_vars, eval_method()) + remaining,
                     local_vars, allow_recursion)
+                return ret, should_return or should_abort
             else:
-                return eval_method()
+                return eval_method(), should_return
 
-        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
-        if m:
-            fname = m.group('func')
-            argvals = tuple([
-                int(v) if v.isdigit() else local_vars[v]
-                for v in self._separate(m.group('args'))])
+        elif md.get('function'):
+            fname = m.group('fname')
+            argvals = [self.interpret_expression(v, local_vars, allow_recursion)
+                       for v in self._separate(m.group('args'))]
             if fname in local_vars:
-                return local_vars[fname](argvals)
+                return local_vars[fname](argvals, allow_recursion=allow_recursion), should_return
             elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
-            return self._functions[fname](argvals)
+            return self._functions[fname](argvals, allow_recursion=allow_recursion), should_return
 
-        if expr:
-            raise ExtractorError('Unsupported JS expression %r' % expr)
+        raise self.Exception(
+            'Unsupported JS expression ' + (expr[:40] if expr != stmt else ''), expr=stmt)
+
+    def interpret_expression(self, expr, local_vars, allow_recursion):
+        ret, should_return = self.interpret_statement(expr, local_vars, allow_recursion)
+        if should_return:
+            raise self.Exception('Cannot return from an expression', expr)
+        return ret
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -494,15 +602,17 @@ def extract_object(self, objname):
                 }\s*;
             ''' % (re.escape(objname), _FUNC_NAME_RE),
             self.code)
+        if not obj_m:
+            raise self.Exception('Could not find object ' + objname)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
             r'''(?x)
-                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
-            ''' % _FUNC_NAME_RE,
+                (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
+            ''' % (_FUNC_NAME_RE, _NAME_RE),
             fields)
         for f in fields_m:
-            argnames = f.group('args').split(',')
+            argnames = self.build_arglist(f.group('args'))
             obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
 
         return obj
@@ -510,15 +620,19 @@ def extract_object(self, objname):
     def extract_function_code(self, funcname):
         """ @returns argnames, code """
         func_m = re.search(
-            r'''(?x)
-                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
+            r'''(?xs)
+                (?:
+                    function\s+%(name)s|
+                    [{;,]\s*%(name)s\s*=\s*function|
+                    (?:var|const|let)\s+%(name)s\s*=\s*function
+                )\s*
                 \((?P<args>[^)]*)\)\s*
-                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
+                (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
         code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
-            raise ExtractorError('Could not find JS function %r' % funcname)
-        return func_m.group('args').split(','), code
+            raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
+        return self.build_arglist(func_m.group('args')), code
 
     def extract_function(self, funcname):
         return self.extract_function_from_code(*self.extract_function_code(funcname))
@@ -534,7 +648,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [x.strip() for x in mobj.group('args').split(',')],
+                    self.build_arglist(mobj.group('args')),
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)
@@ -542,17 +656,22 @@ def extract_function_from_code(self, argnames, code, *global_stack):
     def call_function(self, funcname, *args):
         return self.extract_function(funcname)(args)
 
+    @classmethod
+    def build_arglist(cls, arg_text):
+        if not arg_text:
+            return []
+        return list(filter(None, (x.strip() or None for x in cls._separate(arg_text))))
+
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]
-        local_vars = global_stack.pop(0)
-
-        def resf(args, **kwargs):
-            local_vars.update(dict(zip(argnames, args)))
-            local_vars.update(kwargs)
-            var_stack = LocalNameSpace(local_vars, *global_stack)
-            for stmt in self._separate(code.replace('\n', ''), ';'):
-                ret, should_abort = self.interpret_statement(stmt, var_stack)
-                if should_abort:
-                    break
-            return ret
+        argnames = tuple(argnames)
+
+        def resf(args, kwargs={}, allow_recursion=100):
+            global_stack[0].update(
+                zip_longest(argnames, args, fillvalue=None))
+            global_stack[0].update(kwargs)
+            var_stack = LocalNameSpace(*global_stack)
+            ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)
+            if should_abort:
+                return ret
         return resf
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4e00317f1c7..a5f584ec572 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1696,6 +1696,17 @@ def random_user_agent():
         'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
 }
 
+# Timezone names for RFC2822 obs-zone
+# From https://github.com/python/cpython/blob/3.11/Lib/email/_parseaddr.py#L36-L42
+TIMEZONE_NAMES = {
+    'UT': 0, 'UTC': 0, 'GMT': 0, 'Z': 0,
+    'AST': -4, 'ADT': -3,  # Atlantic (used in Canada)
+    'EST': -5, 'EDT': -4,  # Eastern
+    'CST': -6, 'CDT': -5,  # Central
+    'MST': -7, 'MDT': -6,  # Mountain
+    'PST': -8, 'PDT': -7   # Pacific
+}
+
 KNOWN_EXTENSIONS = (
     'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
     'flv', 'f4v', 'f4a', 'f4b',
@@ -1735,12 +1746,17 @@ def random_user_agent():
     '%b %dth %Y %I:%M',
     '%Y %m %d',
     '%Y-%m-%d',
+    '%Y.%m.%d.',
     '%Y/%m/%d',
     '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
+    '%Y%m%d%H%M',
+    '%Y%m%d%H%M%S',
+    '%Y%m%d',
     '%Y-%m-%d %H:%M',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',
+    '%Y-%m-%d %H:%M:%S:%f',
     '%d.%m.%Y %H:%M',
     '%d.%m.%Y %H.%M',
     '%Y-%m-%dT%H:%M:%SZ',
@@ -1753,6 +1769,7 @@ def random_user_agent():
     '%b %d %Y at %H:%M:%S',
     '%B %d %Y at %H:%M',
     '%B %d %Y at %H:%M:%S',
+    '%H:%M %d-%b-%Y',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
@@ -1763,6 +1780,7 @@ def random_user_agent():
     '%d/%m/%Y',
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
+    '%d-%m-%Y %H:%M',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
@@ -2966,10 +2984,22 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
 
 def extract_timezone(date_str):
     m = re.search(
-        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
+        r'''(?x)
+            ^.{8,}?                                              # >=8 char non-TZ prefix, if present
+            (?P<tz>Z|                                            # just the UTC Z, or
+                (?:(?<=.\b\d{4}|\b\d{2}:\d\d)|                   # preceded by 4 digits or hh:mm or
+                   (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d))     # not preceded by 3 alpha word or >= 4 alpha or 2 digits
+                   [ ]?                                          # optional space
+                (?P<sign>\+|-)                                   # +/-
+                (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})       # hh[:]mm
+            $)
+        ''', date_str)
     if not m:
-        timezone = datetime.timedelta()
+        m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
+        timezone = TIMEZONE_NAMES.get(m and m.group('tz').strip())
+        if timezone is not None:
+            date_str = date_str[:-len(m.group('tz'))]
+        timezone = datetime.timedelta(hours=timezone or 0)
     else:
         date_str = date_str[:-len(m.group('tz'))]
         if not m.group('sign'):
@@ -3037,7 +3067,8 @@ def unified_timestamp(date_str, day_first=True):
     if date_str is None:
         return None
 
-    date_str = re.sub(r'[,|]', '', date_str)
+    date_str = re.sub(r'\s+', ' ', re.sub(
+        r'(?i)[,|]|(mon|tues?|wed(nes)?|thu(rs)?|fri|sat(ur)?)(day)?', '', date_str))
 
     pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)
@@ -3063,7 +3094,7 @@ def unified_timestamp(date_str, day_first=True):
             pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
-        return calendar.timegm(timetuple) + pm_delta * 3600
+        return calendar.timegm(timetuple) + pm_delta * 3600 - timezone.total_seconds()
 
 
 def determine_ext(url, default_ext='unknown_video'):
@@ -3673,13 +3704,11 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)
-    if v == '':
-        v = None
-    if v is None:
+    if v in (None, ''):
         return default
     try:
         return int(v) * invscale // scale
-    except (ValueError, TypeError):
+    except (ValueError, TypeError, OverflowError):
         return default
 
 

From 18b7043b0cb28ba614293bac901be769c2c56a72 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 15 Aug 2022 11:58:04 -0700
Subject: [PATCH 363/394] Backport from yt-dlp

---
 youtube_dl/extractor/parler.py | 74 +++++++++++++++-------------------
 1 file changed, 32 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/parler.py b/youtube_dl/extractor/parler.py
index 96a955b4d78..e00562e08f7 100644
--- a/youtube_dl/extractor/parler.py
+++ b/youtube_dl/extractor/parler.py
@@ -1,12 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from urllib import parse
-
 from .common import InfoExtractor
-
 from ..utils import (
     clean_html,
+    int_or_none,
     strip_or_none,
     unified_timestamp,
     urlencode_postdata,
@@ -14,10 +12,8 @@
 
 
 class ParlerIE(InfoExtractor):
-    """Extract videos from posts on Parler."""
-
-    _UUID_RE = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
-    _VALID_URL = r'https://parler\.com/feed/(?P<id>%s)' % (_UUID_RE, )
+    IE_DESC = 'Posts on parler.com'
+    _VALID_URL = r'https://parler\.com/feed/(?P<id>[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})'
     _TESTS = [
         {
             'url': 'https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7',
@@ -25,12 +21,17 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'df79fdba-07cc-48fe-b085-3293897520d7',
                 'ext': 'mp4',
-                'title': 'TulsiGabbard-720',
-                'description': 'Puberty-blocking procedures promoted by the Biden/Harris Admin are child abuse. The FDA has recently confirmed these hormones/drugs have extremely dangerous side effects, like brain swelling and vision loss.',
+                'thumbnail': 'https://bl-images.parler.com/videos/6ce7cdf3-a27a-4d72-bf9c-d3e17ce39a66/thumbnail.jpeg',
+                'title': 'Parler video #df79fdba-07cc-48fe-b085-3293897520d7',
+                'description': 'md5:6f220bde2df4a97cbb89ac11f1fd8197',
                 'timestamp': 1659744000,
                 'upload_date': '20220806',
                 'uploader': 'Tulsi Gabbard',
                 'uploader_id': 'TulsiGabbard',
+                'uploader_url': 'https://parler.com/TulsiGabbard',
+                'view_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
         {
@@ -39,49 +40,38 @@ class ParlerIE(InfoExtractor):
             'info_dict': {
                 'id': 'a7406eb4-91e5-4793-b5e3-ade57a24e287',
                 'ext': 'mp4',
-                'title': 'BennyJohnson-360',
+                'thumbnail': 'https://bl-images.parler.com/videos/317827a8-1e48-4cbc-981f-7dd17d4c1183/thumbnail.jpeg',
+                'title': 'Parler video #a7406eb4-91e5-4793-b5e3-ade57a24e287',
                 'description': 'This man should run for office',
                 'timestamp': 1659657600,
                 'upload_date': '20220805',
                 'uploader': 'Benny Johnson',
                 'uploader_id': 'BennyJohnson',
+                'uploader_url': 'https://parler.com/BennyJohnson',
+                'view_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        # Get data from API
-        api_url = 'https://parler.com/open-api/ParleyDetailEndpoint.php'
-        payload = urlencode_postdata({'uuid': video_id})
-        status = self._download_json(api_url, video_id, data=payload)
-
-        # Pull out video
-        url = status['data'][0]['primary']['video_data']['videoSrc']
-        # now we know this exists and is a dict
-        data = status['data'][0]['primary']
-
-        # Pull out metadata
-        description = strip_or_none(clean_html(data.get('full_body')))
-        timestamp = unified_timestamp(data.get('date_created'))
-        uploader = strip_or_none(data.get('name'))
-        uploader_id = strip_or_none(data.get('username'))
-        uploader_url = ('https://parler.com/' + uploader_id) if uploader_id else None
-
-        # Keep the file name short so it doesn't exceed filesystem limits
-        title = self._generic_title(url)
-        if uploader_id:
-            title = '%s-%s' % (uploader_id, title)
-
-        # Return the result
+        data = self._download_json(
+            'https://parler.com/open-api/ParleyDetailEndpoint.php', video_id,
+            data=urlencode_postdata({'uuid': video_id}))['data'][0]
+        primary = data['primary']
         return {
             'id': video_id,
-            'url': url,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'uploader_url': uploader_url,
-        }
+            'url': primary['video_data']['videoSrc'],
+            'thumbnail': primary['video_data']['thumbnailUrl'],
+            'title': "Parler video #%s" % video_id,
+            'description': strip_or_none(clean_html(primary.get('full_body'))) or None,
+            'timestamp': unified_timestamp(primary.get('date_created')),
+            'uploader': strip_or_none(primary.get('name')),
+            'uploader_id': strip_or_none(primary.get('username')),
+            'uploader_url': 'https://parler.com/%s' % strip_or_none(primary.get('username')),
+            'view_count': int_or_none(primary.get('view_count')),
+            'comment_count': int_or_none(data['engagement']['commentCount']),
+            'repost_count': int_or_none(data['engagement']['echoCount']),
+        }
\ No newline at end of file

From 38e5559877e3342d5795d6a4e22e356a91cc1463 Mon Sep 17 00:00:00 2001
From: palewire <b@palewi.re>
Date: Mon, 15 Aug 2022 12:04:19 -0700
Subject: [PATCH 364/394] Backport update

---
 youtube_dl/extractor/truth.py | 50 +++++++++--------------------------
 1 file changed, 12 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/truth.py b/youtube_dl/extractor/truth.py
index 1d8a671a226..cde353c76ea 100644
--- a/youtube_dl/extractor/truth.py
+++ b/youtube_dl/extractor/truth.py
@@ -2,19 +2,16 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
 from ..utils import (
     clean_html,
     int_or_none,
+    strip_or_none,
     unified_timestamp,
-    strip_or_none
 )
 
 
 class TruthIE(InfoExtractor):
-    """Extract videos from posts on Donald Trump's truthsocial.com."""
-
-    _VALID_URL = r'https://truthsocial\.com/@[^/]+/posts/(?P<id>[\d]+)'
+    _VALID_URL = r'https?://truthsocial\.com/@[^/]+/posts/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://truthsocial.com/@realDonaldTrump/posts/108779000807761862',
@@ -22,7 +19,8 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108779000807761862',
                 'ext': 'qt',
-                'title': 'Donald J. Trump - 0d8691160c73d663',
+                'title': 'Truth video #108779000807761862',
+                'description': None,
                 'timestamp': 1659835827,
                 'upload_date': '20220807',
                 'uploader': 'Donald J. Trump',
@@ -39,7 +37,7 @@ class TruthIE(InfoExtractor):
             'info_dict': {
                 'id': '108618228543962049',
                 'ext': 'mp4',
-                'title': 'md5:48813a16498d21b07edf24e1af621e83',
+                'title': 'md5:debde7186cf83f60ff7b44dbb9444e35',
                 'description': 'md5:e070ba6bcf6165957e26a7a94ef6d975',
                 'timestamp': 1657382637,
                 'upload_date': '20220709',
@@ -52,44 +50,20 @@ class TruthIE(InfoExtractor):
             },
         },
     ]
-    _GEO_COUNTRIES = ['US']  # The site is only available in the US
 
     def _real_extract(self, url):
-        # Get data from API
         video_id = self._match_id(url)
-        status = self._download_json(
-            'https://truthsocial.com/api/v1/statuses/' + video_id,
-            video_id
-        )
-
-        # Pull out video
-        url = status['media_attachments'][0]['url']
-
-        # Return the stuff
-        account = status.get('account') or {}
-        uploader = strip_or_none(account.get('display_name'))
-        uploader_id = strip_or_none(account.get('username'))
-        post = strip_or_none(clean_html(status.get('content')))
-
-        # Set the title, handling case where its too long or empty
-        if len(post) > 40:
-            title = post[:35] + "[...]"
-        elif len(post) == 0:
-            title = self._generic_title(url)
-        else:
-            title = post
-        if uploader:
-            title = '%s - %s' % (uploader, title)
-
+        status = self._download_json(f'https://truthsocial.com/api/v1/statuses/{video_id}', video_id)
+        uploader_id = strip_or_none(status['account']['username'])
         return {
             'id': video_id,
-            'url': url,
-            'title': title,
-            'description': post,
+            'url': status['media_attachments'][0]['url'],
+            'title': 'Truth video #%s' % video_id,
+            'description': strip_or_none(clean_html(status.get('content'))) or None,
             'timestamp': unified_timestamp(status.get('created_at')),
-            'uploader': uploader,
+            'uploader': strip_or_none(status['account']['display_name']),
             'uploader_id': uploader_id,
-            'uploader_url': ('https://truthsocial.com/@' + uploader_id) if uploader_id else None,
+            'uploader_url': 'https://truthsocial.com/@%s' % uploader_id,
             'repost_count': int_or_none(status.get('reblogs_count')),
             'like_count': int_or_none(status.get('favourites_count')),
             'comment_count': int_or_none(status.get('replies_count')),

From df0ebb9c64b030bd1849f1ad3718bb28e47d4db1 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 26 Aug 2022 18:56:50 +0200
Subject: [PATCH 365/394] cleaning

---
 docs/supportedsites.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c1219e495a9..16977f7208b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -713,6 +713,7 @@
  - **ParamountNetwork**
  - **ParamountPlus**
  - **ParamountPlusSeries**
+ - **Parler**: Posts on parler.com
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
@@ -1015,6 +1016,7 @@
  - **TRT**: TRT (Turkish State TV) programs and series
  - **TRTLive**: TRT (Turkish State TV and radio) live channels
  - **TruNews**
+ - **Truth**
  - **TruTV**
  - **Tube8**
  - **TubiTv**

From 2faf013fbfcc284379e1e6659e9b5f0876b45973 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Fri, 26 Aug 2022 19:00:27 +0200
Subject: [PATCH 366/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 19460dfdbb0..03a9e5cc4c5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.08.16'
+__version__ = '2022.08.26'

From e24c9760351835fafe942ab31f272f7ee9196f2f Mon Sep 17 00:00:00 2001
From: Nick Payne <Nick.Payne@anuvu.com>
Date: Sun, 28 Aug 2022 13:31:25 -0400
Subject: [PATCH 367/394] [bandlabextractor] added new bandlab extractor for
 track, album, and playlist

---
 youtube_dl/extractor/bandlab.py    | 192 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 193 insertions(+)
 create mode 100644 youtube_dl/extractor/bandlab.py

diff --git a/youtube_dl/extractor/bandlab.py b/youtube_dl/extractor/bandlab.py
new file mode 100644
index 00000000000..dcf8c15a287
--- /dev/null
+++ b/youtube_dl/extractor/bandlab.py
@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BandlabIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bandlab\.com/post/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'https://www.bandlab.com/post/f5f04998635a44ea819cacdba7ae2076_f8d8574c3bdaec11b6562818783151a1',
+        'info_dict': {
+            'id': 'f5f04998635a44ea819cacdba7ae2076_f8d8574c3bdaec11b6562818783151a1',
+            'ext': 'm4a',
+            'title': 'ON MY OWN (unreleased)',
+            'artist': 'Michael MacDonald',
+        },
+    }
+
+    def _real_extract(self, url):
+        track_id = self._match_id(url)
+        config = self._download_json(
+            'http://www.bandlab.com/api/v1.3/posts/%s' % track_id, track_id)
+        track_url = config['track']['sample']['audioUrl']
+
+        return {
+            'id': track_id,
+            'title': config['track']['name'],
+            'url': track_url,
+            'artist': config['creator']['name']
+        }
+
+
+class BandlabAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bandlab\.com/[^/]+/albums/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.bandlab.com/sbsdasani/albums/dc26e307-e51f-ed11-95d7-002248452390',
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '91feeb36-8e10-4c91-ae57-ffac0a98c6b4',
+                    'title': 'How\'d I Lose You? (Intro)',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': 'd87c50a2-70cb-4937-9b97-3ae8646ca3aa',
+                    'title': 'Money $$$',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': 'ff2909ff-348f-448d-9d2c-7edbf2f0ec5e',
+                    'title': 'You\'ll Be Mine',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '53786c38-1f3c-4921-9271-793a64af7186',
+                    'title': 'Who You Are',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '0394bb27-113f-4d19-b902-f9c1fe6ba8a8',
+                    'title': 'In Your Eyes',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '2aa44689-c7fa-4d0e-b28a-e0d1dd570372',
+                    'title': 'The Same',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '281fe589-a559-4260-802d-78a6c7a973d8',
+                    'title': 'Fall In Love',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '83a9dc0a-702f-40fd-82f6-ab847f6a7b46',
+                    'title': 'Tried So Hard',
+                    'ext': 'm4a',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '3f6a4a1c-eb73-449f-bd45-f7958d6f2de1',
+                    'title': 'The End Of Everything',
+                    'ext': 'm4a',
+                },
+            }
+        ],
+        'info_dict': {
+            'id': 'dc26e307-e51f-ed11-95d7-002248452390',
+            'album': 'ENDLESS SUMMER',
+            'artist': 'Michael MacDonald'
+        },
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+        config = self._download_json(
+            'http://www.bandlab.com/api/v1.3/albums/%s' % album_id, album_id)
+        tracks = config['posts']
+        entries = []
+        for track in tracks:
+            if 'track' in track:
+                url = track['track']['sample']['audioUrl']
+                name = track['track']['name']
+            elif 'revision' in track:
+                url = track['revision']['mixdown']['file']
+                name = track['revision']['song']['name']
+            else:
+                raise Exception("Neither track nor revision found in 'posts' object")
+            id_regex = r'.+/(?P<id>[^/]+).m4a'
+            id = re.compile(id_regex).match(url).group('id')
+            entries.append({
+                'url': url,
+                'id': id,
+                'title': name
+            })
+
+        return {
+            '_type': 'playlist',
+            'id': album_id,
+            'album': config['name'],
+            'artist': config['creator']['name'],
+            'entries': entries
+        }
+
+
+class BandlabPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bandlab\.com/[^/]+/collections/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.bandlab.com/hexatetrahedronx/collections/606442da-f426-ed11-9441-000d3a3f83b4',
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '3086d9cd-983a-4d2c-b088-adc3eb8d618d',
+                    'title': u'If You Leave (En-Esp) Si Te Vas #forkable #fork 🎤🎹📀',
+                    'ext': 'm4a'
+                }
+            }
+        ],
+        'info_dict': {
+            'id': '606442da-f426-ed11-9441-000d3a3f83b4',
+            'playlist': 'youtube-mp3-test'
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        config = self._download_json(
+            'http://www.bandlab.com/api/v1.3/collections/%s' % playlist_id, playlist_id)
+        tracks = config['posts']
+        entries = []
+        for track in tracks:
+            if 'track' in track:
+                url = track['track']['sample']['audioUrl']
+                name = track['track']['name']
+            elif 'revision' in track:
+                url = track['revision']['mixdown']['file']
+                name = track['revision']['song']['name']
+            else:
+                raise Exception("Neither track nor revision found in 'posts' object")
+            id_regex = r'.+/(?P<id>[^/]+).m4a'
+            id = re.compile(id_regex).match(url).group('id')
+            entries.append({
+                'url': url,
+                'id': id,
+                'title': name
+            })
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'playlist': config['name'],
+            'entries': entries
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b61a..3f9acd0ec85 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -98,6 +98,7 @@
 from .baidu import BaiduVideoIE
 from .bandaichannel import BandaiChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
+from .bandlab import BandlabIE, BandlabAlbumIE, BandlabPlaylistIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,

From 0201ee10820ac40dd30b8721322aa077364bada9 Mon Sep 17 00:00:00 2001
From: Nick Payne <Nick.Payne@anuvu.com>
Date: Mon, 29 Aug 2022 12:03:03 -0400
Subject: [PATCH 368/394] remove utf8 song name in test for bandlab playlist
 extractor

---
 youtube_dl/extractor/bandlab.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bandlab.py b/youtube_dl/extractor/bandlab.py
index dcf8c15a287..b14686ccfb4 100644
--- a/youtube_dl/extractor/bandlab.py
+++ b/youtube_dl/extractor/bandlab.py
@@ -1,7 +1,3 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import unicode_literals
-
 import re
 
 from .common import InfoExtractor
@@ -145,19 +141,19 @@ def _real_extract(self, url):
 class BandlabPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bandlab\.com/[^/]+/collections/(?P<id>[^/]+)'
     _TEST = {
-        'url': 'https://www.bandlab.com/hexatetrahedronx/collections/606442da-f426-ed11-9441-000d3a3f83b4',
+        'url': 'https://www.bandlab.com/hexatetrahedronx/collections/8fb1041c-e865-eb11-9889-0050f28a2802',
         'playlist': [
             {
                 'info_dict': {
-                    'id': '3086d9cd-983a-4d2c-b088-adc3eb8d618d',
-                    'title': u'If You Leave (En-Esp) Si Te Vas #forkable #fork 🎤🎹📀',
+                    'id': '8f37e4aa-92c4-eb11-a7ad-0050f280467f',
+                    'title': 'psych ward',
                     'ext': 'm4a'
                 }
             }
         ],
         'info_dict': {
-            'id': '606442da-f426-ed11-9441-000d3a3f83b4',
-            'playlist': 'youtube-mp3-test'
+            'id': '8fb1041c-e865-eb11-9889-0050f28a2802',
+            'playlist': 'DOOMTAPE'
         }
     }
 

From 6dd45b8d4378d0403e6ca9b380673b85ae78c572 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 29 Aug 2022 19:53:51 +0100
Subject: [PATCH 369/394] Add Unicode compatibility header

---
 youtube_dl/extractor/bandlab.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bandlab.py b/youtube_dl/extractor/bandlab.py
index b14686ccfb4..c3b2dd7d9f2 100644
--- a/youtube_dl/extractor/bandlab.py
+++ b/youtube_dl/extractor/bandlab.py
@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor

From 3fe2483e31e394befed644855e992c6d74035941 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sun, 11 Sep 2022 19:39:55 +0200
Subject: [PATCH 370/394] cleaning

---
 README.md              | 3 ++-
 docs/supportedsites.md | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cd2d36993f4..8f0f01b227d 100644
--- a/README.md
+++ b/README.md
@@ -438,7 +438,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          necessary (currently supported:
                                          mp4|flv|ogg|webm|mkv|avi)
     --postprocessor-args ARGS            Give these arguments to the
-                                         postprocessor
+                                         postprocessor (if postprocessing is
+                                         required)
     -k, --keep-video                     Keep the video file on disk after the
                                          post-processing; the video is erased by
                                          default
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 16977f7208b..7032201bd99 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -90,6 +90,9 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **Bandcamp:weekly**
+ - **Bandlab**
+ - **BandlabAlbum**
+ - **BandlabPlaylist**
  - **bangumi.bilibili.com**: BiliBili番剧
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer

From 93fdde431c0ff0cb8eb8a738f2629734b66b527e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 21:05:00 +0100
Subject: [PATCH 371/394] [test, etc] Improve download test logs; also clean up
 some new flake8 issues (#31153)

* [test] Identify testcase errors better
* [test] Identify download errors better
* [extractor/minds] Linter
* [extractor/aes] Linter
---
 test/test_download.py         | 7 +++++--
 youtube_dl/aes.py             | 2 +-
 youtube_dl/extractor/minds.py | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 6a6673bc2ea..19936969fa0 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -33,6 +33,7 @@
 from youtube_dl.utils import (
     DownloadError,
     ExtractorError,
+    error_to_compat_str,
     format_bytes,
     UnavailableVideoError,
 )
@@ -108,7 +109,7 @@ def print_skipping(reason):
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
             if not (info_dict.get('id') and info_dict.get('ext')):
-                raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
+                raise Exception('Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file' % (tname, ))
 
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
@@ -161,7 +162,9 @@ def try_rm_tcs_files(tcs=None):
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
-                        raise
+                        msg = getattr(err, 'msg', error_to_compat_str(err))
+                        err.msg = '%s (%s)' % (msg, tname, )
+                        raise err
 
                     if try_num == RETRIES:
                         report_warning('%s failed due to network errors, skipping...' % tname)
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 461bb6d413a..d0de2d93f39 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -303,7 +303,7 @@ def xor(data1, data2):
 
 
 def rijndael_mul(a, b):
-    if(a == 0 or b == 0):
+    if (a == 0 or b == 0):
         return 0
     return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
 
diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py
index 8e9f0f8254e..e8fd582aa97 100644
--- a/youtube_dl/extractor/minds.py
+++ b/youtube_dl/extractor/minds.py
@@ -78,7 +78,7 @@ def _real_extract(self, url):
             else:
                 return self.url_result(entity['perma_url'])
         else:
-            assert(entity['subtype'] == 'video')
+            assert (entity['subtype'] == 'video')
             video_id = entity_id
         # 1080p and webm formats available only on the sources array
         video = self._call_api(

From c9234b1cceafeddb10387321ecbc9328d150aaba Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Aug 2022 15:37:59 +0100
Subject: [PATCH 372/394] [core] Make `--max-downloads ...` stop immediately on
 reaching the limit

Based on and closes #26638.
---
 youtube_dl/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3895b408fc0..e77b8d50cff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1779,10 +1779,9 @@ def process_info(self, info_dict):
 
         assert info_dict.get('_type', 'video') == 'video'
 
-        max_downloads = self.params.get('max_downloads')
-        if max_downloads is not None:
-            if self._num_downloads >= int(max_downloads):
-                raise MaxDownloadsReached()
+        max_downloads = int_or_none(self.params.get('max_downloads')) or float('inf')
+        if self._num_downloads >= max_downloads:
+            raise MaxDownloadsReached()
 
         # TODO: backward compatibility, to be removed
         info_dict['fulltitle'] = info_dict['title']
@@ -2062,6 +2061,9 @@ def compatible_formats(formats):
                     self.report_error('postprocessing: %s' % str(err))
                     return
                 self.record_download_archive(info_dict)
+                # avoid possible nugatory search for further items (PR #26638)
+                if self._num_downloads >= max_downloads:
+                    raise MaxDownloadsReached()
 
     def download(self, url_list):
         """Download a given list of URLs."""

From 27281eb5bb87a150f41d3c74edb589a5bf010efa Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 14 Aug 2022 18:45:45 +0100
Subject: [PATCH 373/394] [jsinterp] Overhaul JSInterp to handle new YT players
 4c3f79c5, 324f67b9 (#31170)

* back-port from yt-dlp 8f53dc44a0cc1c2d98c35740b9293462c080f5d0, thanks pukkandan
* also support void, improve <</>> precedence, improve expressions in comma-list
* add more tests
---
 test/test_jsinterp.py          |  49 ++-
 test/test_utils.py             |   3 +
 test/test_youtube_signature.py |  13 +
 youtube_dl/compat.py           |  54 ++-
 youtube_dl/jsinterp.py         | 589 ++++++++++++++++++++-------------
 youtube_dl/utils.py            |  47 ++-
 6 files changed, 504 insertions(+), 251 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index acdabffb1f8..c6c93174353 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ def test_basic(self):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('function x3(){42}')
+        self.assertEqual(jsi.call_function('x3'), None)
+
         jsi = JSInterpreter('var x5 = function(){return 42;}')
         self.assertEqual(jsi.call_function('x5'), 42)
 
@@ -51,8 +54,11 @@ def test_operators(self):
         jsi = JSInterpreter('function f(){return 11 >> 2;}')
         self.assertEqual(jsi.call_function('f'), 2)
 
+        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
+        self.assertEqual(jsi.call_function('f'), 5)
+
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
 
     def test_parens(self):
@@ -62,6 +68,10 @@ def test_parens(self):
         jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
         self.assertEqual(jsi.call_function('f'), 9)
 
+    def test_quotes(self):
+        jsi = JSInterpreter(r'function f(){return "a\"\\("}')
+        self.assertEqual(jsi.call_function('f'), r'a"\(')
+
     def test_assignments(self):
         jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
         self.assertEqual(jsi.call_function('f'), 31)
@@ -104,18 +114,29 @@ def test_precedence(self):
         }''')
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
+    def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 86000)
+        jsi = JSInterpreter('''
+        function x(dt) { return new Date(dt) - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+
     def test_call(self):
         jsi = JSInterpreter('''
         function x() { return 2; }
-        function y(a) { return x() + a; }
+        function y(a) { return x() + (a?a:0); }
         function z() { return y(3); }
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
+        self.assertEqual(jsi.call_function('y'), 2)
 
     def test_for_loop(self):
         # function x() { a=0; for (i=0; i-10; i++) {a++} a }
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
@@ -156,19 +177,19 @@ def test_try(self):
 
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_for_loop_break(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_literal_list(self):
         jsi = JSInterpreter('''
-        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
         ''')
         self.assertEqual(jsi.call_function('x'), [5, 6, 7])
 
@@ -177,6 +198,22 @@ def test_comma(self):
         function x() { a=5; a -= 1, a+=3; return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
+        jsi = JSInterpreter('''
+        function x() { a=5; return (a -= 1, a+=3, a); }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
+    def test_void(self):
+        jsi = JSInterpreter('''
+        function x() { return void 42; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+    def test_return_function(self):
+        jsi = JSInterpreter('''
+        function x() { return [1, function(){return 1}][1] }
+        ''')
+        self.assertEqual(jsi.call_function('x')([]), 1)
 
 
 if __name__ == '__main__':
diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e1e..f1a748ddee8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,9 @@ def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
         self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
         self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
+        self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1)
+        self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86)
+        self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index fc5e9828e2c..6e955e0f019 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -90,12 +90,25 @@
         'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
         'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
     ),
+    (
+        'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
+        'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
+    ),
+    (
+        'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
+        'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
+    ),
+    (
+        'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
+        'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
+    ),
 ]
 
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
         PLAYER_URLS = (
+            ('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9f5f85daeaa..6d2c31a61c5 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2985,7 +2985,6 @@ def unpack(self, string):
     except ImportError:
         compat_filter = filter
 
-
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -2995,6 +2994,57 @@ def unpack(self, string):
         compat_zip = zip
 
 
+# method renamed between Py2/3
+try:
+    from itertools import zip_longest as compat_itertools_zip_longest
+except ImportError:
+    from itertools import izip_longest as compat_itertools_zip_longest
+
+
+# new class in collections
+try:
+    from collections import ChainMap as compat_collections_chain_map
+except ImportError:
+    # Py < 3.3
+    class compat_collections_chain_map(compat_collections_abc.MutableMapping):
+
+        maps = [{}]
+
+        def __init__(self, *maps):
+            self.maps = list(maps) or [{}]
+
+        def __getitem__(self, k):
+            for m in self.maps:
+                if k in m:
+                    return m[k]
+            raise KeyError(k)
+
+        def __setitem__(self, k, v):
+            self.maps[0].__setitem__(k, v)
+            return
+
+        def __delitem__(self, k):
+            if k in self.maps[0]:
+                del self.maps[0][k]
+                return
+            raise KeyError(k)
+
+        def __iter__(self):
+            return itertools.chain(*reversed(self.maps))
+
+        def __len__(self):
+            return len(iter(self))
+
+        def new_child(self, m=None, **kwargs):
+            m = m or {}
+            m.update(kwargs)
+            return compat_collections_chain_map(m, *self.maps)
+
+        @property
+        def parents(self):
+            return compat_collections_chain_map(*(self.maps[1:]))
+
+
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
         if isinstance(s, compat_str):
@@ -3031,6 +3081,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_basestring',
     'compat_chr',
     'compat_collections_abc',
+    'compat_collections_chain_map',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
@@ -3051,6 +3102,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_input',
     'compat_integer_types',
     'compat_itertools_count',
+    'compat_itertools_zip_longest',
     'compat_kwargs',
     'compat_map',
     'compat_numeric_types',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8eaa911cdab..c60a9b3c234 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,42 +1,87 @@
 from __future__ import unicode_literals
 
+import itertools
 import json
+import math
 import operator
 import re
 
 from .utils import (
+    NO_DEFAULT,
     ExtractorError,
+    js_to_json,
     remove_quotes,
+    unified_timestamp,
 )
 from .compat import (
-    compat_collections_abc,
+    compat_collections_chain_map as ChainMap,
+    compat_itertools_zip_longest as zip_longest,
     compat_str,
 )
-MutableMapping = compat_collections_abc.MutableMapping
 
+_NAME_RE = r'[a-zA-Z_$][\w$]*'
 
-class Nonlocal:
-    pass
-
+# (op, definition) in order of binding priority, tightest first
+# avoid dict to maintain order
+# definition None => Defined in JSInterpreter._operator
+_DOT_OPERATORS = (
+    ('.', None),
+    # TODO: ('?.', None),
+)
 
-_OPERATORS = [
+_OPERATORS = (
     ('|', operator.or_),
     ('^', operator.xor),
     ('&', operator.and_),
     ('>>', operator.rshift),
     ('<<', operator.lshift),
-    ('-', operator.sub),
     ('+', operator.add),
-    ('%', operator.mod),
-    ('/', operator.truediv),
+    ('-', operator.sub),
     ('*', operator.mul),
-]
-_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
+    ('/', operator.truediv),
+    ('%', operator.mod),
+)
 
-_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+_COMP_OPERATORS = (
+    ('===', operator.is_),
+    ('==', operator.eq),
+    ('!==', operator.is_not),
+    ('!=', operator.ne),
+    ('<=', operator.le),
+    ('>=', operator.ge),
+    ('<', operator.lt),
+    ('>', operator.gt),
+)
+
+_LOG_OPERATORS = (
+    ('&', operator.and_),
+    ('|', operator.or_),
+    ('^', operator.xor),
+)
+
+_SC_OPERATORS = (
+    ('?', None),
+    ('||', None),
+    ('&&', None),
+    # TODO: ('??', None),
+)
+
+_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+_QUOTES = '\'"'
+
+
+def _ternary(cndn, if_true=True, if_false=False):
+    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
+    if cndn in (False, None, 0, ''):
+        return if_false
+    try:
+        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
+            return if_false
+    except TypeError:
+        pass
+    return if_true
 
 
 class JS_Break(ExtractorError):
@@ -49,70 +94,77 @@ def __init__(self):
         ExtractorError.__init__(self, 'Invalid continue')
 
 
-class LocalNameSpace(MutableMapping):
-    def __init__(self, *stack):
-        self.stack = tuple(stack)
-
-    def __getitem__(self, key):
-        for scope in self.stack:
-            if key in scope:
-                return scope[key]
-        raise KeyError(key)
-
+class LocalNameSpace(ChainMap):
     def __setitem__(self, key, value):
-        for scope in self.stack:
+        for scope in self.maps:
             if key in scope:
                 scope[key] = value
-                break
-        else:
-            self.stack[0][key] = value
-        return value
+                return
+        self.maps[0][key] = value
 
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
-    def __iter__(self):
-        for scope in self.stack:
-            for scope_item in iter(scope):
-                yield scope_item
-
-    def __len__(self, key):
-        return len(iter(self))
-
     def __repr__(self):
-        return 'LocalNameSpace%s' % (self.stack, )
+        return 'LocalNameSpace%s' % (self.maps, )
 
 
 class JSInterpreter(object):
+    __named_object_counter = 0
+
     def __init__(self, code, objects=None):
-        if objects is None:
-            objects = {}
-        self.code = code
-        self._functions = {}
-        self._objects = objects
-        self.__named_object_counter = 0
+        self.code, self._functions = code, {}
+        self._objects = {} if objects is None else objects
+
+    class Exception(ExtractorError):
+        def __init__(self, msg, *args, **kwargs):
+            expr = kwargs.pop('expr', None)
+            if expr is not None:
+                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
+            super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
-        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
         namespace[name] = obj
         return name
 
     @staticmethod
-    def _separate(expr, delim=',', max_split=None):
+    def _separate(expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
+        start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
+        in_quote, escaping = None, False
         for idx, char in enumerate(expr):
-            if char in _MATCHING_PARENS:
-                counters[_MATCHING_PARENS[char]] += 1
-            elif char in counters:
-                counters[char] -= 1
-            if char != delim[pos] or any(counters.values()):
-                pos = 0
+            if not in_quote:
+                if char in _MATCHING_PARENS:
+                    counters[_MATCHING_PARENS[char]] += 1
+                elif char in counters:
+                    counters[char] -= 1
+            if not escaping:
+                if char in _QUOTES and in_quote in (char, None):
+                    in_quote = None if in_quote else char
+                else:
+                    escaping = in_quote and char == '\\'
+            else:
+                escaping = False
+
+            if char != delim[pos] or any(counters.values()) or in_quote:
+                pos = skipping = 0
                 continue
-            elif pos != delim_len:
+            elif skipping > 0:
+                skipping -= 1
+                continue
+            elif pos == 0 and skip_delims:
+                here = expr[idx:]
+                for s in skip_delims if isinstance(skip_delims, (list, tuple)) else [skip_delims]:
+                    if here.startswith(s) and s:
+                        skipping = len(s) - 1
+                        break
+                if skipping > 0:
+                    continue
+            if pos < delim_len:
                 pos += 1
                 continue
             yield expr[start: idx - delim_len]
@@ -122,61 +174,108 @@ def _separate(expr, delim=',', max_split=None):
                 break
         yield expr[start:]
 
-    @staticmethod
-    def _separate_at_paren(expr, delim):
-        separated = list(JSInterpreter._separate(expr, delim, 1))
+    @classmethod
+    def _separate_at_paren(cls, expr, delim):
+        separated = list(cls._separate(expr, delim, 1))
+
         if len(separated) < 2:
-            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
+    @staticmethod
+    def _all_operators():
+        return itertools.chain(
+            _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
+
+    def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
+        if op in ('||', '&&'):
+            if (op == '&&') ^ _ternary(left_val):
+                return left_val  # short circuiting
+        elif op == '?':
+            right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
+
+        right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
+        opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
+        if not opfunc:
+            return right_val
+
+        try:
+            return opfunc(left_val, right_val)
+        except Exception as e:
+            raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
+
+    def _index(self, obj, idx):
+        if idx == 'length':
+            return len(obj)
+        try:
+            return obj[int(idx)] if isinstance(obj, list) else obj[idx]
+        except Exception as e:
+            raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
+
+    def _dump(self, obj, namespace):
+        try:
+            return json.dumps(obj)
+        except TypeError:
+            return self._named_object(namespace, obj)
+
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
-            raise ExtractorError('Recursion limit reached')
+            raise self.Exception('Recursion limit reached')
+        allow_recursion -= 1
 
-        sub_statements = list(self._separate(stmt, ';'))
-        stmt = (sub_statements or ['']).pop()
+        should_return = False
+        sub_statements = list(self._separate(stmt, ';')) or ['']
+        expr = stmt = sub_statements.pop().strip()
         for sub_stmt in sub_statements:
-            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
-            if should_abort:
-                return ret
+            ret, should_return = self.interpret_statement(sub_stmt, local_vars, allow_recursion)
+            if should_return:
+                return ret, should_return
 
-        should_abort = False
-        stmt = stmt.lstrip()
-        stmt_m = re.match(r'var\s', stmt)
-        if stmt_m:
-            expr = stmt[len(stmt_m.group(0)):]
-        else:
-            return_m = re.match(r'return(?:\s+|$)', stmt)
-            if return_m:
-                expr = stmt[len(return_m.group(0)):]
-                should_abort = True
-            else:
-                # Try interpreting it as an expression
-                expr = stmt
+        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
+        if m:
+            expr = stmt[len(m.group(0)):].strip()
+            should_return = not m.group('var')
+        if not expr:
+            return None, should_return
 
-        v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return v, should_abort
+        if expr[0] in _QUOTES:
+            inner, outer = self._separate(expr, expr[0], 1)
+            inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
+            if not outer:
+                return inner, should_return
+            expr = self._named_object(local_vars, inner) + outer
+
+        if expr.startswith('new '):
+            obj = expr[4:]
+            if obj.startswith('Date('):
+                left, right = self._separate_at_paren(obj[4:], ')')
+                left = self.interpret_expression(left, local_vars, allow_recursion)
+                expr = unified_timestamp(left, False)
+                if not expr:
+                    raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
+                expr = self._dump(int(expr * 1000), local_vars) + right
+            else:
+                raise self.Exception('Unsupported object {obj}'.format(**locals()), expr=expr)
 
-    def interpret_expression(self, expr, local_vars, allow_recursion):
-        expr = expr.strip()
-        if expr == '':  # Empty expression
-            return None
+        if expr.startswith('void '):
+            left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
+            return None, should_return
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
-            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
-                return inner
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
             inner, outer = self._separate_at_paren(expr, ')')
-            inner = self.interpret_expression(inner, local_vars, allow_recursion)
-            if not outer:
-                return inner
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
+            if not outer or should_abort:
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('['):
             inner, outer = self._separate_at_paren(expr, ']')
@@ -185,57 +284,53 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'try\s*', expr)
-        if m:
+        m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('try'):
             if expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
-            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
             if should_abort:
-                return ret
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                return ret, True
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
-        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
-        md = m.groupdict() if m else {}
-        if md.get('catch'):
+        elif md.get('catch'):
             # We ignore the catch block
             _, expr = self._separate_at_paren(expr, '}')
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('for'):
-            def raise_constructor_error(c):
-                raise ExtractorError(
-                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
-
             constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             if remaining.startswith('{'):
                 body, expr = self._separate_at_paren(remaining, '}')
             else:
-                m = re.match(r'switch\s*\(', remaining)  # FIXME
-                if m:
-                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if switch_m:
+                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
                     body, expr = self._separate_at_paren(remaining, '}')
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
                     body, expr = remaining, ''
             start, cndn, increment = self._separate(constructor, ';')
-            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
-                raise_constructor_error(constructor)
+            self.interpret_expression(start, local_vars, allow_recursion)
             while True:
-                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                     break
                 try:
-                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
                     if should_abort:
-                        return ret
+                        return ret, True
                 except JS_Break:
                     break
                 except JS_Continue:
                     pass
-                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
-                    raise_constructor_error(constructor)
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                self.interpret_expression(increment, local_vars, allow_recursion)
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('switch'):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
@@ -245,7 +340,7 @@ def raise_constructor_error(c):
             for default in (False, True):
                 matched = False
                 for item in items:
-                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    case, stmt = (i.strip() for i in self._separate(item, ':', 1))
                     if default:
                         matched = matched or case == 'default'
                     elif not matched:
@@ -254,24 +349,28 @@ def raise_constructor_error(c):
                     if not matched:
                         continue
                     try:
-                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
                 if matched:
                     break
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         # Comma separated statements
         sub_expressions = list(self._separate(expr))
-        expr = sub_expressions.pop().strip() if sub_expressions else ''
-        for sub_expr in sub_expressions:
-            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+        if len(sub_expressions) > 1:
+            for sub_expr in sub_expressions:
+                ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+            return ret, False
 
         for m in re.finditer(r'''(?x)
-                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
-                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+                (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
+                (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)'''.format(**globals()), expr):
             var = m.group('var1') or m.group('var2')
             start, end = m.span()
             sign = m.group('pre_sign') or m.group('post_sign')
@@ -279,85 +378,87 @@ def raise_constructor_error(c):
             local_vars[var] += 1 if sign[0] == '+' else -1
             if m.group('pre_sign'):
                 ret = local_vars[var]
-            expr = expr[:start] + json.dumps(ret) + expr[end:]
-
-        for op, opfunc in _ASSIGN_OPERATORS:
-            m = re.match(r'''(?x)
-                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
-                \s*%s
-                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
-            if not m:
-                continue
-            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
-
-            if m.groupdict().get('index'):
-                lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
-                if not isinstance(idx, int):
-                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
-                cur = lvar[idx]
-                val = opfunc(cur, right_val)
-                lvar[idx] = val
-                return val
-            else:
-                cur = local_vars.get(m.group('out'))
-                val = opfunc(cur, right_val)
-                local_vars[m.group('out')] = val
-                return val
+            expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
 
-        if expr.isdigit():
-            return int(expr)
-
-        if expr == 'break':
+        if not expr:
+            return None, should_return
+
+        m = re.match(r'''(?x)
+            (?P<assign>
+                (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
+                (?P<op>{_OPERATOR_RE})?
+                =(?P<expr>.*)$
+            )|(?P<return>
+                (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
+            )|(?P<indexing>
+                (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
+            )|(?P<attribute>
+                (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+            )|(?P<function>
+                (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
+            )'''.format(**globals()), expr)
+        md = m.groupdict() if m else {}
+        if md.get('assign'):
+            left_val = local_vars.get(m.group('out'))
+
+            if not m.group('index'):
+                local_vars[m.group('out')] = self._operator(
+                    m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
+                return local_vars[m.group('out')], should_return
+            elif left_val is None:
+                raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
+
+            idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+            if not isinstance(idx, (int, float)):
+                raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
+            idx = int(idx)
+            left_val[idx] = self._operator(
+                m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
+            return left_val[idx], should_return
+
+        elif expr.isdigit():
+            return int(expr), should_return
+
+        elif expr == 'break':
             raise JS_Break()
         elif expr == 'continue':
             raise JS_Continue()
 
-        var_m = re.match(
-            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
-            expr)
-        if var_m:
-            return local_vars[var_m.group('name')]
+        elif md.get('return'):
+            return local_vars[m.group('name')], should_return
 
         try:
-            return json.loads(expr)
+            ret = json.loads(js_to_json(expr))  # strict=True)
+            if not md.get('attribute'):
+                return ret, should_return
         except ValueError:
             pass
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
-        if m:
+        if md.get('indexing'):
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
-            return val[idx]
-
-        def raise_expr_error(where, op, exp):
-            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+            return self._index(val, idx), should_return
 
-        for op, opfunc in _OPERATORS:
-            separated = list(self._separate(expr, op))
+        for op, _ in self._all_operators():
+            # hackety: </> have higher priority than <</>>, but don't confuse them
+            skip_delim = (op + op) if op in ('<', '>') else None
+            separated = list(self._separate(expr, op, skip_delims=skip_delim))
             if len(separated) < 2:
                 continue
-            right_val = separated.pop()
-            left_val = op.join(separated)
-            left_val, should_abort = self.interpret_statement(
-                left_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('left-side', op, expr)
-            right_val, should_abort = self.interpret_statement(
-                right_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('right-side', op, expr)
-            return opfunc(left_val or 0, right_val)
 
-        m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
-            expr)
-        if m:
-            variable = m.group('var')
-            nl = Nonlocal()
+            right_expr = separated.pop()
+            while op == '-' and len(separated) > 1 and not separated[-1].strip():
+                right_expr = '-' + right_expr
+                separated.pop()
+            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
+            return self._operator(op, 0 if left_val is None else left_val,
+                                  right_expr, expr, local_vars, allow_recursion), should_return
 
-            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+        if md.get('attribute'):
+            variable = m.group('var')
+            member = m.group('member')
+            if not member:
+                member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
             if arg_str.startswith('('):
                 arg_str, remaining = self._separate_at_paren(arg_str, ')')
@@ -367,25 +468,24 @@ def raise_expr_error(where, op, exp):
             def assertion(cndn, msg):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
-                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+                    raise ExtractorError('{member} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
-                # nonlocal member
-                member = nl.member
-                if variable == 'String':
-                    obj = compat_str
-                elif variable in local_vars:
-                    obj = local_vars[variable]
-                else:
+                if (variable, member) == ('console', 'debug'):
+                    return
+                types = {
+                    'String': compat_str,
+                    'Math': float,
+                }
+                obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
+                if obj is NO_DEFAULT:
                     if variable not in self._objects:
                         self._objects[variable] = self.extract_object(variable)
                     obj = self._objects[variable]
 
+                # Member access
                 if arg_str is None:
-                    # Member access
-                    if member == 'length':
-                        return len(obj)
-                    return obj[member]
+                    return self._index(obj, member)
 
                 # Function call
                 argvals = [
@@ -396,12 +496,17 @@ def eval_method():
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
-                    raise ExtractorError('Unsupported string method %s' % (member, ))
+                    raise self.Exception('Unsupported string method ' + member, expr=expr)
+                elif obj == float:
+                    if member == 'pow':
+                        assertion(len(argvals) == 2, 'takes two arguments')
+                        return argvals[0] ** argvals[1]
+                    raise self.Exception('Unsupported Math method ' + member, expr=expr)
 
                 if member == 'split':
                     assertion(argvals, 'takes one or more arguments')
-                    assertion(argvals == [''], 'with arguments is not implemented')
-                    return list(obj)
+                    assertion(len(argvals) == 1, 'with limit argument is not implemented')
+                    return obj.split(argvals[0]) if argvals[0] else list(obj)
                 elif member == 'join':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(len(argvals) == 1, 'takes exactly one argument')
@@ -447,7 +552,7 @@ def eval_method():
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
                     f, this = (argvals + [''])[:2]
-                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                    return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
                 elif member == 'indexOf':
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
@@ -457,32 +562,35 @@ def eval_method():
                     except ValueError:
                         return -1
 
-                if isinstance(obj, list):
-                    member = int(member)
-                    nl.member = member
-                return obj[member](argvals)
+                idx = int(member) if isinstance(obj, list) else member
+                return obj[idx](argvals, allow_recursion=allow_recursion)
 
             if remaining:
-                return self.interpret_expression(
+                ret, should_abort = self.interpret_statement(
                     self._named_object(local_vars, eval_method()) + remaining,
                     local_vars, allow_recursion)
+                return ret, should_return or should_abort
             else:
-                return eval_method()
+                return eval_method(), should_return
 
-        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
-        if m:
-            fname = m.group('func')
-            argvals = tuple([
-                int(v) if v.isdigit() else local_vars[v]
-                for v in self._separate(m.group('args'))])
+        elif md.get('function'):
+            fname = m.group('fname')
+            argvals = [self.interpret_expression(v, local_vars, allow_recursion)
+                       for v in self._separate(m.group('args'))]
             if fname in local_vars:
-                return local_vars[fname](argvals)
+                return local_vars[fname](argvals, allow_recursion=allow_recursion), should_return
             elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
-            return self._functions[fname](argvals)
+            return self._functions[fname](argvals, allow_recursion=allow_recursion), should_return
 
-        if expr:
-            raise ExtractorError('Unsupported JS expression %r' % expr)
+        raise self.Exception(
+            'Unsupported JS expression ' + (expr[:40] if expr != stmt else ''), expr=stmt)
+
+    def interpret_expression(self, expr, local_vars, allow_recursion):
+        ret, should_return = self.interpret_statement(expr, local_vars, allow_recursion)
+        if should_return:
+            raise self.Exception('Cannot return from an expression', expr)
+        return ret
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -494,15 +602,17 @@ def extract_object(self, objname):
                 }\s*;
             ''' % (re.escape(objname), _FUNC_NAME_RE),
             self.code)
+        if not obj_m:
+            raise self.Exception('Could not find object ' + objname)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
             r'''(?x)
-                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
-            ''' % _FUNC_NAME_RE,
+                (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
+            ''' % (_FUNC_NAME_RE, _NAME_RE),
             fields)
         for f in fields_m:
-            argnames = f.group('args').split(',')
+            argnames = self.build_arglist(f.group('args'))
             obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
 
         return obj
@@ -510,15 +620,19 @@ def extract_object(self, objname):
     def extract_function_code(self, funcname):
         """ @returns argnames, code """
         func_m = re.search(
-            r'''(?x)
-                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
+            r'''(?xs)
+                (?:
+                    function\s+%(name)s|
+                    [{;,]\s*%(name)s\s*=\s*function|
+                    (?:var|const|let)\s+%(name)s\s*=\s*function
+                )\s*
                 \((?P<args>[^)]*)\)\s*
-                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
+                (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
         code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
-            raise ExtractorError('Could not find JS function %r' % funcname)
-        return func_m.group('args').split(','), code
+            raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
+        return self.build_arglist(func_m.group('args')), code
 
     def extract_function(self, funcname):
         return self.extract_function_from_code(*self.extract_function_code(funcname))
@@ -534,7 +648,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [x.strip() for x in mobj.group('args').split(',')],
+                    self.build_arglist(mobj.group('args')),
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)
@@ -542,17 +656,22 @@ def extract_function_from_code(self, argnames, code, *global_stack):
     def call_function(self, funcname, *args):
         return self.extract_function(funcname)(args)
 
+    @classmethod
+    def build_arglist(cls, arg_text):
+        if not arg_text:
+            return []
+        return list(filter(None, (x.strip() or None for x in cls._separate(arg_text))))
+
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]
-        local_vars = global_stack.pop(0)
-
-        def resf(args, **kwargs):
-            local_vars.update(dict(zip(argnames, args)))
-            local_vars.update(kwargs)
-            var_stack = LocalNameSpace(local_vars, *global_stack)
-            for stmt in self._separate(code.replace('\n', ''), ';'):
-                ret, should_abort = self.interpret_statement(stmt, var_stack)
-                if should_abort:
-                    break
-            return ret
+        argnames = tuple(argnames)
+
+        def resf(args, kwargs={}, allow_recursion=100):
+            global_stack[0].update(
+                zip_longest(argnames, args, fillvalue=None))
+            global_stack[0].update(kwargs)
+            var_stack = LocalNameSpace(*global_stack)
+            ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)
+            if should_abort:
+                return ret
         return resf
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4e00317f1c7..a5f584ec572 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1696,6 +1696,17 @@ def random_user_agent():
         'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
 }
 
+# Timezone names for RFC2822 obs-zone
+# From https://github.com/python/cpython/blob/3.11/Lib/email/_parseaddr.py#L36-L42
+TIMEZONE_NAMES = {
+    'UT': 0, 'UTC': 0, 'GMT': 0, 'Z': 0,
+    'AST': -4, 'ADT': -3,  # Atlantic (used in Canada)
+    'EST': -5, 'EDT': -4,  # Eastern
+    'CST': -6, 'CDT': -5,  # Central
+    'MST': -7, 'MDT': -6,  # Mountain
+    'PST': -8, 'PDT': -7   # Pacific
+}
+
 KNOWN_EXTENSIONS = (
     'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
     'flv', 'f4v', 'f4a', 'f4b',
@@ -1735,12 +1746,17 @@ def random_user_agent():
     '%b %dth %Y %I:%M',
     '%Y %m %d',
     '%Y-%m-%d',
+    '%Y.%m.%d.',
     '%Y/%m/%d',
     '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
+    '%Y%m%d%H%M',
+    '%Y%m%d%H%M%S',
+    '%Y%m%d',
     '%Y-%m-%d %H:%M',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',
+    '%Y-%m-%d %H:%M:%S:%f',
     '%d.%m.%Y %H:%M',
     '%d.%m.%Y %H.%M',
     '%Y-%m-%dT%H:%M:%SZ',
@@ -1753,6 +1769,7 @@ def random_user_agent():
     '%b %d %Y at %H:%M:%S',
     '%B %d %Y at %H:%M',
     '%B %d %Y at %H:%M:%S',
+    '%H:%M %d-%b-%Y',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
@@ -1763,6 +1780,7 @@ def random_user_agent():
     '%d/%m/%Y',
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
+    '%d-%m-%Y %H:%M',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
@@ -2966,10 +2984,22 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
 
 def extract_timezone(date_str):
     m = re.search(
-        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
+        r'''(?x)
+            ^.{8,}?                                              # >=8 char non-TZ prefix, if present
+            (?P<tz>Z|                                            # just the UTC Z, or
+                (?:(?<=.\b\d{4}|\b\d{2}:\d\d)|                   # preceded by 4 digits or hh:mm or
+                   (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d))     # not preceded by 3 alpha word or >= 4 alpha or 2 digits
+                   [ ]?                                          # optional space
+                (?P<sign>\+|-)                                   # +/-
+                (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})       # hh[:]mm
+            $)
+        ''', date_str)
     if not m:
-        timezone = datetime.timedelta()
+        m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
+        timezone = TIMEZONE_NAMES.get(m and m.group('tz').strip())
+        if timezone is not None:
+            date_str = date_str[:-len(m.group('tz'))]
+        timezone = datetime.timedelta(hours=timezone or 0)
     else:
         date_str = date_str[:-len(m.group('tz'))]
         if not m.group('sign'):
@@ -3037,7 +3067,8 @@ def unified_timestamp(date_str, day_first=True):
     if date_str is None:
         return None
 
-    date_str = re.sub(r'[,|]', '', date_str)
+    date_str = re.sub(r'\s+', ' ', re.sub(
+        r'(?i)[,|]|(mon|tues?|wed(nes)?|thu(rs)?|fri|sat(ur)?)(day)?', '', date_str))
 
     pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)
@@ -3063,7 +3094,7 @@ def unified_timestamp(date_str, day_first=True):
             pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
-        return calendar.timegm(timetuple) + pm_delta * 3600
+        return calendar.timegm(timetuple) + pm_delta * 3600 - timezone.total_seconds()
 
 
 def determine_ext(url, default_ext='unknown_video'):
@@ -3673,13 +3704,11 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)
-    if v == '':
-        v = None
-    if v is None:
+    if v in (None, ''):
         return default
     try:
         return int(v) * invscale // scale
-    except (ValueError, TypeError):
+    except (ValueError, TypeError, OverflowError):
         return default
 
 

From 6b98eb46fcb5e5a505a2bbe0460b49b1cfc84c89 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 15 Aug 2022 16:45:04 +0100
Subject: [PATCH 374/394] [postprocessor] Don't replace existing value with
 null metadata parsed from title

---
 youtube_dl/postprocessor/metadatafromtitle.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index f5c14d974f4..6cd5bb70f3a 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -40,6 +40,8 @@ def run(self, info):
                 % self._titleformat)
             return [], info
         for attribute, value in match.groupdict().items():
+            if value is None:
+                continue
             info[attribute] = value
             self._downloader.to_screen(
                 '[fromtitle] parsed %s: %s'

From 1862bf568bcc1f4f18e19e46838c641066961f96 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 17 Aug 2022 14:22:02 +0100
Subject: [PATCH 375/394] [jsinterp] Improve JS language support (#31175)

* operator ??
* operator ?.
* operator **
* accurate operator functions
* `undefined` handling
* object literals {a: 1, "b": expr}
* more tests for weird JS comparisons: see https://github.com/ytdl-org/youtube-dl/issues/31173#issuecomment-1217854397.
---
 test/test_jsinterp.py          | 114 ++++++++++++++++++++
 test/test_youtube_signature.py |   4 +
 youtube_dl/jsinterp.py         | 189 ++++++++++++++++++++++++++-------
 3 files changed, 267 insertions(+), 40 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c6c93174353..328941e09cc 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -8,7 +8,10 @@
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import math
+
 from youtube_dl.jsinterp import JSInterpreter
+undefined = JSInterpreter.undefined
 
 
 class TestJSInterpreter(unittest.TestCase):
@@ -48,6 +51,9 @@ def test_operators(self):
         jsi = JSInterpreter('function f(){return 1 << 5;}')
         self.assertEqual(jsi.call_function('f'), 32)
 
+        jsi = JSInterpreter('function f(){return 2 ** 5}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
         jsi = JSInterpreter('function f(){return 19 & 21;}')
         self.assertEqual(jsi.call_function('f'), 17)
 
@@ -57,6 +63,15 @@ def test_operators(self):
         jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
         self.assertEqual(jsi.call_function('f'), 5)
 
+        jsi = JSInterpreter('function f(){return 1 == 2}')
+        self.assertEqual(jsi.call_function('f'), False)
+
+        jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+        jsi = JSInterpreter('function f(){return 0 ?? 42;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
     def test_array_access(self):
         jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
@@ -203,6 +218,11 @@ def test_comma(self):
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
 
+        jsi = JSInterpreter('''
+        function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
     def test_void(self):
         jsi = JSInterpreter('''
         function x() { return void 42; }
@@ -215,6 +235,100 @@ def test_return_function(self):
         ''')
         self.assertEqual(jsi.call_function('x')([]), 1)
 
+    def test_null(self):
+        jsi = JSInterpreter('''
+        function x() { return null; }
+        ''')
+        self.assertIs(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { return [null > 0, null < 0, null == 0, null === 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [null >= 0, null <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True])
+
+    def test_undefined(self):
+        jsi = JSInterpreter('''
+        function x() { return undefined === undefined; }
+        ''')
+        self.assertTrue(jsi.call_function('x'))
+
+        jsi = JSInterpreter('''
+        function x() { return undefined; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined >= 0, undefined <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, True, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
+        ''')
+        for y in jsi.call_function('x'):
+            self.assertTrue(math.isnan(y))
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v**0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 1)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, undefined, undefined])
+
+        jsi = JSInterpreter('function x(){return undefined ?? 42; }')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_object(self):
+        jsi = JSInterpreter('''
+        function x() { return {}; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), {})
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [42, 0])
+        jsi = JSInterpreter('''
+        function x() { let a; return a?.qq; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 6e955e0f019..4d756dad308 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -102,6 +102,10 @@
         'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
         'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
     ),
+    (
+        'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
+        'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c60a9b3c234..8e119d08a3b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -7,7 +7,6 @@
 import re
 
 from .utils import (
-    NO_DEFAULT,
     ExtractorError,
     js_to_json,
     remove_quotes,
@@ -21,6 +20,70 @@
 
 _NAME_RE = r'[a-zA-Z_$][\w$]*'
 
+_UNDEFINED = object()
+
+
+def _js_bit_op(op):
+
+    def wrapped(a, b):
+        def zeroise(x):
+            return 0 if x in (None, _UNDEFINED) else x
+        return op(zeroise(a), zeroise(b))
+
+    return wrapped
+
+
+def _js_arith_op(op):
+
+    def wrapped(a, b):
+        if _UNDEFINED in (a, b):
+            return float('nan')
+        return op(a or 0, b or 0)
+
+    return wrapped
+
+
+def _js_div(a, b):
+    if _UNDEFINED in (a, b) or not (a and b):
+        return float('nan')
+    return float('inf') if not b else operator.truediv(a or 0, b)
+
+
+def _js_mod(a, b):
+    if _UNDEFINED in (a, b) or not b:
+        return float('nan')
+    return (a or 0) % b
+
+
+def _js_exp(a, b):
+    if not b:
+        # even 0 ** 0 !!
+        return 1
+    if _UNDEFINED in (a, b):
+        return float('nan')
+    return (a or 0) ** b
+
+
+def _js_eq_op(op):
+
+    def wrapped(a, b):
+        if set((a, b)) <= set((None, _UNDEFINED)):
+            return op(a, a)
+        return op(a, b)
+
+    return wrapped
+
+
+def _js_comp_op(op):
+
+    def wrapped(a, b):
+        if _UNDEFINED in (a, b):
+            return False
+        return op(a or 0, b or 0)
+
+    return wrapped
+
+
 # (op, definition) in order of binding priority, tightest first
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
@@ -30,40 +93,38 @@
 )
 
 _OPERATORS = (
-    ('|', operator.or_),
-    ('^', operator.xor),
-    ('&', operator.and_),
-    ('>>', operator.rshift),
-    ('<<', operator.lshift),
-    ('+', operator.add),
-    ('-', operator.sub),
-    ('*', operator.mul),
-    ('/', operator.truediv),
-    ('%', operator.mod),
+    ('>>', _js_bit_op(operator.rshift)),
+    ('<<', _js_bit_op(operator.lshift)),
+    ('+', _js_arith_op(operator.add)),
+    ('-', _js_arith_op(operator.sub)),
+    ('*', _js_arith_op(operator.mul)),
+    ('/', _js_div),
+    ('%', _js_mod),
+    ('**', _js_exp),
 )
 
 _COMP_OPERATORS = (
     ('===', operator.is_),
-    ('==', operator.eq),
+    ('==', _js_eq_op(operator.eq)),
     ('!==', operator.is_not),
-    ('!=', operator.ne),
-    ('<=', operator.le),
-    ('>=', operator.ge),
-    ('<', operator.lt),
-    ('>', operator.gt),
+    ('!=', _js_eq_op(operator.ne)),
+    ('<=', _js_comp_op(operator.le)),
+    ('>=', _js_comp_op(operator.ge)),
+    ('<', _js_comp_op(operator.lt)),
+    ('>', _js_comp_op(operator.gt)),
 )
 
 _LOG_OPERATORS = (
-    ('&', operator.and_),
-    ('|', operator.or_),
-    ('^', operator.xor),
+    ('|', _js_bit_op(operator.or_)),
+    ('^', _js_bit_op(operator.xor)),
+    ('&', _js_bit_op(operator.and_)),
 )
 
 _SC_OPERATORS = (
     ('?', None),
+    ('??', None),
     ('||', None),
     ('&&', None),
-    # TODO: ('??', None),
 )
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
@@ -74,7 +135,7 @@
 
 def _ternary(cndn, if_true=True, if_false=False):
     """Simulate JS's ternary operator (cndn?if_true:if_false)"""
-    if cndn in (False, None, 0, ''):
+    if cndn in (False, None, 0, '', _UNDEFINED):
         return if_false
     try:
         if math.isnan(cndn):  # NB: NaN cannot be checked by membership
@@ -95,6 +156,12 @@ def __init__(self):
 
 
 class LocalNameSpace(ChainMap):
+    def __getitem__(self, key):
+        try:
+            return super(LocalNameSpace, self).__getitem__(key)
+        except KeyError:
+            return _UNDEFINED
+
     def __setitem__(self, key, value):
         for scope in self.maps:
             if key in scope:
@@ -105,6 +172,13 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
+    def __contains__(self, key):
+        try:
+            super(LocalNameSpace, self).__getitem__(key)
+            return True
+        except KeyError:
+            return False
+
     def __repr__(self):
         return 'LocalNameSpace%s' % (self.maps, )
 
@@ -112,6 +186,8 @@ def __repr__(self):
 class JSInterpreter(object):
     __named_object_counter = 0
 
+    undefined = _UNDEFINED
+
     def __init__(self, code, objects=None):
         self.code, self._functions = code, {}
         self._objects = {} if objects is None else objects
@@ -185,12 +261,16 @@ def _separate_at_paren(cls, expr, delim):
     @staticmethod
     def _all_operators():
         return itertools.chain(
+            # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
             _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
             if (op == '&&') ^ _ternary(left_val):
                 return left_val  # short circuiting
+        elif op == '??':
+            if left_val not in (None, self.undefined):
+                return left_val
         elif op == '?':
             right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
 
@@ -204,12 +284,14 @@ def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion)
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
 
-    def _index(self, obj, idx):
+    def _index(self, obj, idx, allow_undefined=False):
         if idx == 'length':
             return len(obj)
         try:
             return obj[int(idx)] if isinstance(obj, list) else obj[idx]
         except Exception as e:
+            if allow_undefined:
+                return self.undefined
             raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
@@ -249,8 +331,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             obj = expr[4:]
             if obj.startswith('Date('):
                 left, right = self._separate_at_paren(obj[4:], ')')
-                left = self.interpret_expression(left, local_vars, allow_recursion)
-                expr = unified_timestamp(left, False)
+                expr = unified_timestamp(
+                    self.interpret_expression(left, local_vars, allow_recursion), False)
                 if not expr:
                     raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
                 expr = self._dump(int(expr * 1000), local_vars) + right
@@ -263,6 +345,14 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
+            # try for object expression
+            sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
+            if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
+                return dict(
+                    (key_expr if re.match(_NAME_RE, key_expr) else key_expr,
+                     self.interpret_expression(val_expr, local_vars, allow_recursion))
+                    for key_expr, val_expr in sub_expressions), should_return
+            # or statement list
             inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
                 return inner, should_abort or should_return
@@ -387,13 +477,13 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             (?P<assign>
                 (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
                 (?P<op>{_OPERATOR_RE})?
-                =(?P<expr>.*)$
+                =(?!=)(?P<expr>.*)$
             )|(?P<return>
                 (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
             )|(?P<indexing>
                 (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
             )|(?P<attribute>
-                (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+                (?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
             )|(?P<function>
                 (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
             )'''.format(**globals()), expr)
@@ -405,7 +495,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 local_vars[m.group('out')] = self._operator(
                     m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
                 return local_vars[m.group('out')], should_return
-            elif left_val is None:
+            elif left_val in (None, self.undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
             idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
@@ -424,6 +514,9 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         elif expr == 'continue':
             raise JS_Continue()
 
+        elif expr == 'undefined':
+            return self.undefined, should_return
+
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
 
@@ -441,7 +534,9 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
 
         for op, _ in self._all_operators():
             # hackety: </> have higher priority than <</>>, but don't confuse them
-            skip_delim = (op + op) if op in ('<', '>') else None
+            skip_delim = (op + op) if op in '<>*?' else None
+            if op == '?':
+                skip_delim = (skip_delim, '?.')
             separated = list(self._separate(expr, op, skip_delims=skip_delim))
             if len(separated) < 2:
                 continue
@@ -451,12 +546,10 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 right_expr = '-' + right_expr
                 separated.pop()
             left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
-            return self._operator(op, 0 if left_val is None else left_val,
-                                  right_expr, expr, local_vars, allow_recursion), should_return
+            return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
 
         if md.get('attribute'):
-            variable = m.group('var')
-            member = m.group('member')
+            variable, member, nullish = m.group('var', 'member', 'nullish')
             if not member:
                 member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
@@ -477,15 +570,24 @@ def eval_method():
                     'String': compat_str,
                     'Math': float,
                 }
-                obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
-                if obj is NO_DEFAULT:
-                    if variable not in self._objects:
-                        self._objects[variable] = self.extract_object(variable)
-                    obj = self._objects[variable]
+                obj = local_vars.get(variable)
+                if obj in (self.undefined, None):
+                    obj = types.get(variable, self.undefined)
+                if obj is self.undefined:
+                    try:
+                        if variable not in self._objects:
+                            self._objects[variable] = self.extract_object(variable)
+                        obj = self._objects[variable]
+                    except self.Exception:
+                        if not nullish:
+                            raise
+
+                if nullish and obj is self.undefined:
+                    return self.undefined
 
                 # Member access
                 if arg_str is None:
-                    return self._index(obj, member)
+                    return self._index(obj, member, nullish)
 
                 # Function call
                 argvals = [
@@ -660,7 +762,14 @@ def call_function(self, funcname, *args):
     def build_arglist(cls, arg_text):
         if not arg_text:
             return []
-        return list(filter(None, (x.strip() or None for x in cls._separate(arg_text))))
+
+        def valid_arg(y):
+            y = y.strip()
+            if not y:
+                raise cls.Exception('Missing arg in "%s"' % (arg_text, ))
+            return y
+
+        return [valid_arg(x) for x in cls._separate(arg_text)]
 
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]

From a63bbcbfcf66ce6bc9ba4b793e1dc4412a0cd559 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 11:45:04 +0100
Subject: [PATCH 376/394] [jsinterp] Handle regexp literals and throw/catch
 execution (#31182)

* based on https://github.com/yt-dlp/yt-dlp/commit/f6ca640b122239d5ab215f8c2564efb7ac3e8c65, thanks pukkandan
* adds parse support for regexp flags
---
 test/test_jsinterp.py          |  21 +++++
 test/test_youtube_signature.py |   4 +
 youtube_dl/jsinterp.py         | 136 +++++++++++++++++++++++++++------
 3 files changed, 139 insertions(+), 22 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 328941e09cc..faddf00d5a6 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -9,6 +9,7 @@
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
+import re
 
 from youtube_dl.jsinterp import JSInterpreter
 undefined = JSInterpreter.undefined
@@ -316,19 +317,39 @@ def test_object(self):
         function x() { return {}; }
         ''')
         self.assertEqual(jsi.call_function('x'), {})
+
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
         ''')
         self.assertEqual(jsi.call_function('x'), [42, 0])
+
         jsi = JSInterpreter('''
         function x() { let a; return a?.qq; }
         ''')
         self.assertIs(jsi.call_function('x'), undefined)
+
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
         ''')
         self.assertIs(jsi.call_function('x'), undefined)
 
+    def test_regex(self):
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; }
+        ''')
+        self.assertIs(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; return a; }
+        ''')
+        # Pythons disagree on the type of a pattern
+        self.assertTrue(isinstance(jsi.call_function('x'), type(re.compile(''))))
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/i; return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4d756dad308..43e22388d0b 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -106,6 +106,10 @@
         'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
         'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
     ),
+    (
+        'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
+        'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8e119d08a3b..48c27a1c04b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -7,6 +7,7 @@
 import re
 
 from .utils import (
+    error_to_compat_str,
     ExtractorError,
     js_to_json,
     remove_quotes,
@@ -130,7 +131,7 @@ def wrapped(a, b):
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
-_QUOTES = '\'"'
+_QUOTES = '\'"/'
 
 
 def _ternary(cndn, if_true=True, if_false=False):
@@ -155,6 +156,12 @@ def __init__(self):
         ExtractorError.__init__(self, 'Invalid continue')
 
 
+class JS_Throw(ExtractorError):
+    def __init__(self, e):
+        self.error = e
+        ExtractorError.__init__(self, 'Uncaught exception ' + error_to_compat_str(e))
+
+
 class LocalNameSpace(ChainMap):
     def __getitem__(self, key):
         try:
@@ -172,6 +179,17 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
+    # except
+    def pop(self, key, *args):
+        try:
+            off = self.__getitem__(key)
+            super(LocalNameSpace, self).__delitem__(key)
+            return off
+        except KeyError:
+            if len(args) > 0:
+                return args[0]
+            raise
+
     def __contains__(self, key):
         try:
             super(LocalNameSpace, self).__getitem__(key)
@@ -188,9 +206,29 @@ class JSInterpreter(object):
 
     undefined = _UNDEFINED
 
+    RE_FLAGS = {
+        # special knowledge: Python's re flags are bitmask values, current max 128
+        # invent new bitmask values well above that for literal parsing
+        # TODO: new pattern class to execute matches with these flags
+        'd': 1024,  # Generate indices for substring matches
+        'g': 2048,  # Global search
+        'i': re.I,  # Case-insensitive search
+        'm': re.M,  # Multi-line search
+        's': re.S,  # Allows . to match newline characters
+        'u': re.U,  # Treat a pattern as a sequence of unicode code points
+        'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
+    }
+
+    _EXC_NAME = '__youtube_dl_exception__'
+    _OBJ_NAME = '__youtube_dl_jsinterp_obj'
+
+    OP_CHARS = None
+
     def __init__(self, code, objects=None):
         self.code, self._functions = code, {}
         self._objects = {} if objects is None else objects
+        if type(self).OP_CHARS is None:
+            type(self).OP_CHARS = self.OP_CHARS = self.__op_chars()
 
     class Exception(ExtractorError):
         def __init__(self, msg, *args, **kwargs):
@@ -199,32 +237,64 @@ def __init__(self, msg, *args, **kwargs):
                 msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
+    @classmethod
+    def __op_chars(cls):
+        op_chars = set(';,')
+        for op in cls._all_operators():
+            for c in op[0]:
+                op_chars.add(c)
+        return op_chars
+
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
-        name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
+        name = '%s%d' % (self._OBJ_NAME, self.__named_object_counter)
         namespace[name] = obj
         return name
 
-    @staticmethod
-    def _separate(expr, delim=',', max_split=None, skip_delims=None):
+    @classmethod
+    def _regex_flags(cls, expr):
+        flags = 0
+        if not expr:
+            return flags, expr
+        for idx, ch in enumerate(expr):
+            if ch not in cls.RE_FLAGS:
+                break
+            flags |= cls.RE_FLAGS[ch]
+        return flags, expr[idx:] if idx > 0 else expr
+
+    @classmethod
+    def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
-        in_quote, escaping = None, False
+        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
+        in_quote, escaping, skipping = None, False, 0
+        after_op, in_regex_char_group, skip_re = True, False, 0
+
         for idx, char in enumerate(expr):
+            if skip_re > 0:
+                skip_re -= 1
+                continue
             if not in_quote:
                 if char in _MATCHING_PARENS:
                     counters[_MATCHING_PARENS[char]] += 1
                 elif char in counters:
                     counters[char] -= 1
-            if not escaping:
-                if char in _QUOTES and in_quote in (char, None):
-                    in_quote = None if in_quote else char
-                else:
-                    escaping = in_quote and char == '\\'
-            else:
-                escaping = False
+            if not escaping and char in _QUOTES and in_quote in (char, None):
+                if in_quote or after_op or char != '/':
+                    in_quote = None if in_quote and not in_regex_char_group else char
+                    if in_quote is None and char == '/' and delim != '/':
+                        # regexp flags
+                        n_idx = idx + 1
+                        while n_idx < len(expr) and expr[n_idx] in cls.RE_FLAGS:
+                            n_idx += 1
+                        skip_re = n_idx - idx - 1
+                        if skip_re > 0:
+                            continue
+            elif in_quote == '/' and char in '[]':
+                in_regex_char_group = char == '['
+            escaping = not escaping and in_quote and char == '\\'
+            after_op = not in_quote and char in cls.OP_CHARS or (char == ' ' and after_op)
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -313,16 +383,23 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             if should_return:
                 return ret, should_return
 
-        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
+        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$)|(?P<throw>throw\s+)', stmt)
         if m:
             expr = stmt[len(m.group(0)):].strip()
+            if m.group('throw'):
+                raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
             should_return = not m.group('var')
         if not expr:
             return None, should_return
 
         if expr[0] in _QUOTES:
             inner, outer = self._separate(expr, expr[0], 1)
-            inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
+            if expr[0] == '/':
+                flags, _ = self._regex_flags(outer)
+                inner, outer = inner.replace('"', r'\"'), ''
+                inner = re.compile(js_to_json(inner + expr[0]), flags=flags)  # , strict=True))
+            else:
+                inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
             if not outer:
                 return inner, should_return
             expr = self._named_object(local_vars, inner) + outer
@@ -374,22 +451,37 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        m = re.match(r'''(?x)
+            (?P<try>try|finally)\s*|
+            (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
+            (?P<switch>switch)\s*\(|
+            (?P<for>for)\s*\(|'''.format(**globals()), expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
             if expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
-            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
-            if should_abort:
-                return ret, True
+            try:
+                ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+            except JS_Throw as e:
+                local_vars[self._EXC_NAME] = e.error
+            except Exception as e:
+                # XXX: This works for now, but makes debugging future issues very hard
+                local_vars[self._EXC_NAME] = e
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
         elif md.get('catch'):
-            # We ignore the catch block
-            _, expr = self._separate_at_paren(expr, '}')
+            catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            if self._EXC_NAME in local_vars:
+                catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
+                ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
@@ -503,7 +595,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
             idx = int(idx)
             left_val[idx] = self._operator(
-                m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
+                m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
             return left_val[idx], should_return
 
         elif expr.isdigit():

From 9be9fffb25fb26c124f30c36f1a01da91d5fd9a7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 15:34:33 +0100
Subject: [PATCH 377/394] [jsinterp] Clean up and pull yt-dlp style * add
 compat_re_Pattern * improve compat_collections_chain_map * use class
 JS_Undefined * remove unused code

---
 test/test_jsinterp.py          |  20 +++---
 test/test_youtube_signature.py |   3 +-
 youtube_dl/compat.py           |  21 +++++-
 youtube_dl/jsinterp.py         | 123 ++++++++++++---------------------
 4 files changed, 77 insertions(+), 90 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index faddf00d5a6..96786a84c04 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -11,8 +11,9 @@
 import math
 import re
 
-from youtube_dl.jsinterp import JSInterpreter
-undefined = JSInterpreter.undefined
+from youtube_dl.compat import compat_re_Pattern
+
+from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
 
 
 class TestJSInterpreter(unittest.TestCase):
@@ -261,12 +262,12 @@ def test_undefined(self):
         jsi = JSInterpreter('''
         function x() { return undefined; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { let v; return v; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
@@ -307,7 +308,7 @@ def test_undefined(self):
         jsi = JSInterpreter('''
         function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
         ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, undefined, undefined])
+        self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
 
         jsi = JSInterpreter('function x(){return undefined ?? 42; }')
         self.assertEqual(jsi.call_function('x'), 42)
@@ -326,12 +327,12 @@ def test_object(self):
         jsi = JSInterpreter('''
         function x() { let a; return a?.qq; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
     def test_regex(self):
         jsi = JSInterpreter('''
@@ -342,13 +343,12 @@ def test_regex(self):
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/; return a; }
         ''')
-        # Pythons disagree on the type of a pattern
-        self.assertTrue(isinstance(jsi.call_function('x'), type(re.compile(''))))
+        self.assertIsInstance(jsi.call_function('x'), compat_re_Pattern)
 
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/i; return a; }
         ''')
-        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+        self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
 
 if __name__ == '__main__':
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 43e22388d0b..327d4c40d3c 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -12,10 +12,11 @@
 import re
 import string
 
+from youtube_dl.compat import compat_str, compat_urlretrieve
+
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.jsinterp import JSInterpreter
-from youtube_dl.compat import compat_str, compat_urlretrieve
 
 _SIG_TESTS = [
     (
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 6d2c31a61c5..3002109ca52 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3023,18 +3023,34 @@ def __setitem__(self, k, v):
             self.maps[0].__setitem__(k, v)
             return
 
-        def __delitem__(self, k):
+        def __contains__(self, k):
+            return any((k in m) for m in self.maps)
+
+        def __delitem(self, k):
             if k in self.maps[0]:
                 del self.maps[0][k]
                 return
             raise KeyError(k)
 
+        def __delitem__(self, k):
+            self.__delitem(k)
+
         def __iter__(self):
             return itertools.chain(*reversed(self.maps))
 
         def __len__(self):
             return len(iter(self))
 
+        # to match Py3, don't del directly
+        def pop(self, k, *args):
+            if self.__contains__(k):
+                off = self.__getitem__(k)
+                self.__delitem(k)
+                return off
+            elif len(args) > 0:
+                return args[0]
+            raise KeyError(k)
+
         def new_child(self, m=None, **kwargs):
             m = m or {}
             m.update(kwargs)
@@ -3044,6 +3060,8 @@ def new_child(self, m=None, **kwargs):
         def parents(self):
             return compat_collections_chain_map(*(self.maps[1:]))
 
+# Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
+compat_re_Pattern = type(re.compile(''))
 
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
@@ -3110,6 +3128,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_os_name',
     'compat_parse_qs',
     'compat_print',
+    'compat_re_Pattern',
     'compat_realpath',
     'compat_setenv',
     'compat_shlex_quote',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 48c27a1c04b..6719d0dfd77 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -19,16 +19,12 @@
     compat_str,
 )
 
-_NAME_RE = r'[a-zA-Z_$][\w$]*'
-
-_UNDEFINED = object()
-
 
 def _js_bit_op(op):
 
     def wrapped(a, b):
         def zeroise(x):
-            return 0 if x in (None, _UNDEFINED) else x
+            return 0 if x in (None, JS_Undefined) else x
         return op(zeroise(a), zeroise(b))
 
     return wrapped
@@ -37,7 +33,7 @@ def zeroise(x):
 def _js_arith_op(op):
 
     def wrapped(a, b):
-        if _UNDEFINED in (a, b):
+        if JS_Undefined in (a, b):
             return float('nan')
         return op(a or 0, b or 0)
 
@@ -45,22 +41,21 @@ def wrapped(a, b):
 
 
 def _js_div(a, b):
-    if _UNDEFINED in (a, b) or not (a and b):
+    if JS_Undefined in (a, b) or not (a and b):
         return float('nan')
     return float('inf') if not b else operator.truediv(a or 0, b)
 
 
 def _js_mod(a, b):
-    if _UNDEFINED in (a, b) or not b:
+    if JS_Undefined in (a, b) or not b:
         return float('nan')
     return (a or 0) % b
 
 
 def _js_exp(a, b):
     if not b:
-        # even 0 ** 0 !!
-        return 1
-    if _UNDEFINED in (a, b):
+        return 1  # even 0 ** 0 !!
+    elif JS_Undefined in (a, b):
         return float('nan')
     return (a or 0) ** b
 
@@ -68,7 +63,7 @@ def _js_exp(a, b):
 def _js_eq_op(op):
 
     def wrapped(a, b):
-        if set((a, b)) <= set((None, _UNDEFINED)):
+        if set((a, b)) <= set((None, JS_Undefined)):
             return op(a, a)
         return op(a, b)
 
@@ -78,21 +73,28 @@ def wrapped(a, b):
 def _js_comp_op(op):
 
     def wrapped(a, b):
-        if _UNDEFINED in (a, b):
+        if JS_Undefined in (a, b):
             return False
         return op(a or 0, b or 0)
 
     return wrapped
 
 
+def _js_ternary(cndn, if_true=True, if_false=False):
+    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
+    if cndn in (False, None, 0, '', JS_Undefined):
+        return if_false
+    try:
+        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
+            return if_false
+    except TypeError:
+        pass
+    return if_true
+
+
 # (op, definition) in order of binding priority, tightest first
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
-_DOT_OPERATORS = (
-    ('.', None),
-    # TODO: ('?.', None),
-)
-
 _OPERATORS = (
     ('>>', _js_bit_op(operator.rshift)),
     ('<<', _js_bit_op(operator.lshift)),
@@ -130,20 +132,13 @@ def wrapped(a, b):
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
+_NAME_RE = r'[a-zA-Z_$][\w$]*'
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 _QUOTES = '\'"/'
 
 
-def _ternary(cndn, if_true=True, if_false=False):
-    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
-    if cndn in (False, None, 0, '', _UNDEFINED):
-        return if_false
-    try:
-        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
-            return if_false
-    except TypeError:
-        pass
-    return if_true
+class JS_Undefined(object):
+    pass
 
 
 class JS_Break(ExtractorError):
@@ -167,7 +162,7 @@ def __getitem__(self, key):
         try:
             return super(LocalNameSpace, self).__getitem__(key)
         except KeyError:
-            return _UNDEFINED
+            return JS_Undefined
 
     def __setitem__(self, key, value):
         for scope in self.maps:
@@ -179,24 +174,6 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
-    # except
-    def pop(self, key, *args):
-        try:
-            off = self.__getitem__(key)
-            super(LocalNameSpace, self).__delitem__(key)
-            return off
-        except KeyError:
-            if len(args) > 0:
-                return args[0]
-            raise
-
-    def __contains__(self, key):
-        try:
-            super(LocalNameSpace, self).__getitem__(key)
-            return True
-        except KeyError:
-            return False
-
     def __repr__(self):
         return 'LocalNameSpace%s' % (self.maps, )
 
@@ -204,9 +181,7 @@ def __repr__(self):
 class JSInterpreter(object):
     __named_object_counter = 0
 
-    undefined = _UNDEFINED
-
-    RE_FLAGS = {
+    _RE_FLAGS = {
         # special knowledge: Python's re flags are bitmask values, current max 128
         # invent new bitmask values well above that for literal parsing
         # TODO: new pattern class to execute matches with these flags
@@ -257,10 +232,10 @@ def _regex_flags(cls, expr):
         if not expr:
             return flags, expr
         for idx, ch in enumerate(expr):
-            if ch not in cls.RE_FLAGS:
+            if ch not in cls._RE_FLAGS:
                 break
-            flags |= cls.RE_FLAGS[ch]
-        return flags, expr[idx:] if idx > 0 else expr
+            flags |= cls._RE_FLAGS[ch]
+        return flags, expr[idx + 1:]
 
     @classmethod
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
@@ -283,14 +258,6 @@ def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
             if not escaping and char in _QUOTES and in_quote in (char, None):
                 if in_quote or after_op or char != '/':
                     in_quote = None if in_quote and not in_regex_char_group else char
-                    if in_quote is None and char == '/' and delim != '/':
-                        # regexp flags
-                        n_idx = idx + 1
-                        while n_idx < len(expr) and expr[n_idx] in cls.RE_FLAGS:
-                            n_idx += 1
-                        skip_re = n_idx - idx - 1
-                        if skip_re > 0:
-                            continue
             elif in_quote == '/' and char in '[]':
                 in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
@@ -336,13 +303,13 @@ def _all_operators():
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
-            if (op == '&&') ^ _ternary(left_val):
+            if (op == '&&') ^ _js_ternary(left_val):
                 return left_val  # short circuiting
         elif op == '??':
-            if left_val not in (None, self.undefined):
+            if left_val not in (None, JS_Undefined):
                 return left_val
         elif op == '?':
-            right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
+            right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1))
 
         right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
         opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
@@ -361,7 +328,7 @@ def _index(self, obj, idx, allow_undefined=False):
             return obj[int(idx)] if isinstance(obj, list) else obj[idx]
         except Exception as e:
             if allow_undefined:
-                return self.undefined
+                return JS_Undefined
             raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
@@ -395,9 +362,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if expr[0] in _QUOTES:
             inner, outer = self._separate(expr, expr[0], 1)
             if expr[0] == '/':
-                flags, _ = self._regex_flags(outer)
-                inner, outer = inner.replace('"', r'\"'), ''
-                inner = re.compile(js_to_json(inner + expr[0]), flags=flags)  # , strict=True))
+                flags, outer = self._regex_flags(outer)
+                inner = re.compile(inner[1:], flags=flags)  # , strict=True))
             else:
                 inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
             if not outer:
@@ -422,7 +388,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
-            # try for object expression
+            # try for object expression (Map)
             sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
             if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
                 return dict(
@@ -455,7 +421,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             (?P<try>try|finally)\s*|
             (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
             (?P<switch>switch)\s*\(|
-            (?P<for>for)\s*\(|'''.format(**globals()), expr)
+            (?P<for>for)\s*\(|
+            '''.format(**globals()), expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
             if expr[m.end()] == '{':
@@ -500,7 +467,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             start, cndn, increment = self._separate(constructor, ';')
             self.interpret_expression(start, local_vars, allow_recursion)
             while True:
-                if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
+                if not _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                     break
                 try:
                     ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
@@ -587,7 +554,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 local_vars[m.group('out')] = self._operator(
                     m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
                 return local_vars[m.group('out')], should_return
-            elif left_val in (None, self.undefined):
+            elif left_val in (None, JS_Undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
             idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
@@ -607,7 +574,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             raise JS_Continue()
 
         elif expr == 'undefined':
-            return self.undefined, should_return
+            return JS_Undefined, should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
@@ -663,9 +630,9 @@ def eval_method():
                     'Math': float,
                 }
                 obj = local_vars.get(variable)
-                if obj in (self.undefined, None):
-                    obj = types.get(variable, self.undefined)
-                if obj is self.undefined:
+                if obj in (JS_Undefined, None):
+                    obj = types.get(variable, JS_Undefined)
+                if obj is JS_Undefined:
                     try:
                         if variable not in self._objects:
                             self._objects[variable] = self.extract_object(variable)
@@ -674,8 +641,8 @@ def eval_method():
                         if not nullish:
                             raise
 
-                if nullish and obj is self.undefined:
-                    return self.undefined
+                if nullish and obj is JS_Undefined:
+                    return JS_Undefined
 
                 # Member access
                 if arg_str is None:

From baa75fca868b41f4ac6532a2705b427fba515be0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 19:11:08 +0100
Subject: [PATCH 378/394] [uktvplay] Support domain without .uktv

---
 youtube_dl/extractor/uktvplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/uktvplay.py b/youtube_dl/extractor/uktvplay.py
index f28fd514db6..9ef9638cde9 100644
--- a/youtube_dl/extractor/uktvplay.py
+++ b/youtube_dl/extractor/uktvplay.py
@@ -5,7 +5,7 @@
 
 
 class UKTVPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://uktvplay\.uktv\.co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
+    _VALID_URL = r'https?://uktvplay\.(?:uktv\.)?co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://uktvplay.uktv.co.uk/shows/world-at-war/c/200/watch-online/?video=2117008346001',
         'info_dict': {

From 722e31467bf9e7ad323f428676b5e82cb8903675 Mon Sep 17 00:00:00 2001
From: gudata <gudata@users.noreply.github.com>
Date: Fri, 19 Aug 2022 23:00:21 +0300
Subject: [PATCH 379/394] [infoq] Avoid crash if the page has no `mp3Form`

* proposed fix for issue #31131, aligns with yt-dlp

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/infoq.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 0a70a1fb442..60b02b69958 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,6 +1,9 @@
 # coding: utf-8
 
 from __future__ import unicode_literals
+from ..utils import (
+    ExtractorError,
+)
 
 from ..compat import (
     compat_b64decode,
@@ -90,7 +93,11 @@ def _extract_http_video(self, webpage):
         }]
 
     def _extract_http_audio(self, webpage, video_id):
-        fields = self._form_hidden_inputs('mp3Form', webpage)
+        try:
+            fields = self._form_hidden_inputs('mp3Form', webpage)
+        except ExtractorError:
+            fields = {}
+
         http_audio_url = fields.get('filename')
         if not http_audio_url:
             return []

From 43aebf131383d2c94ae5007dc8926f7cb698e1a8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 21 Aug 2022 00:19:19 +0100
Subject: [PATCH 380/394] [utils] Ensure RFC3986 encoding result is unicode

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a5f584ec572..fea38ed32e9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3970,7 +3970,8 @@ def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, compat_str):
         s = s.encode('utf-8')
-    return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
+    # ensure unicode: after quoting, it can always be converted
+    return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]"))
 
 
 def escape_url(url):

From 1b907b46cc33c656569b0478ba4b5309d1c6b32a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 21 Aug 2022 00:21:02 +0100
Subject: [PATCH 381/394] [core] Avoid processing empty format list after
 removing bad formats * also ensure compat encoding of error strings

---
 youtube_dl/YoutubeDL.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e77b8d50cff..8e85465962e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -721,7 +721,7 @@ def prepare_filename(self, info_dict):
                 filename = encodeFilename(filename, True).decode(preferredencoding())
             return sanitize_path(filename)
         except ValueError as err:
-            self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
+            self.report_error('Error in output template: ' + error_to_compat_str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
     def _match_entry(self, info_dict, incomplete):
@@ -1570,9 +1570,6 @@ def sanitize_numeric_fields(info):
         else:
             formats = info_dict['formats']
 
-        if not formats:
-            raise ExtractorError('No video formats found!')
-
         def is_wellformed(f):
             url = f.get('url')
             if not url:
@@ -1585,7 +1582,10 @@ def is_wellformed(f):
             return True
 
         # Filter out malformed formats for better extraction robustness
-        formats = list(filter(is_wellformed, formats))
+        formats = list(filter(is_wellformed, formats or []))
+
+        if not formats:
+            raise ExtractorError('No video formats found!')
 
         formats_dict = {}
 
@@ -2058,7 +2058,7 @@ def compatible_formats(formats):
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
-                    self.report_error('postprocessing: %s' % str(err))
+                    self.report_error('postprocessing: %s' % error_to_compat_str(err))
                     return
                 self.record_download_archive(info_dict)
                 # avoid possible nugatory search for further items (PR #26638)

From 5dabc11e927b3d869c588d6eb4ff10fdc3bf086d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 25 Aug 2022 12:14:59 +0100
Subject: [PATCH 382/394] [YouTube] Improve error check for n-sig processing

---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 91a3b60582c..3d12e2e4a68 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1500,7 +1500,8 @@ def _extract_n_function(self, video_id, player_url):
         return lambda s: jsi.extract_function_from_code(*func_code)([s])
 
     def _n_descramble(self, n_param, player_url, video_id):
-        """Compute the response to YT's "n" parameter challenge
+        """Compute the response to YT's "n" parameter challenge,
+           or None
 
         Args:
         n_param     -- challenge string that is the value of the
@@ -1518,7 +1519,10 @@ def _n_descramble(self, n_param, player_url, video_id):
             if player_id not in self._player_cache:
                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
             func = self._player_cache[player_id]
-            self._player_cache[sig_id] = func(n_param)
+            ret = func(n_param)
+            if ret.startswith('enhanced_except_'):
+                raise ExtractorError('Unhandled exception in decode')
+            self._player_cache[sig_id] = ret
             if self._downloader.params.get('verbose', False):
                 self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
             return self._player_cache[sig_id]
@@ -1539,10 +1543,12 @@ def _unthrottle_format_urls(self, video_id, player_url, formats):
                 continue
             n_param = n_param[-1]
             n_response = self._n_descramble(n_param, player_url, video_id)
-            if n_response:
-                qs['n'] = [n_response]
-                fmt['url'] = compat_urlparse.urlunparse(
-                    parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+            if n_response is None:
+                # give up if descrambling failed
+                break
+            qs['n'] = [n_response]
+            fmt['url'] = compat_urlparse.urlunparse(
+                parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(

From 660cd68133060e577ab0c7a3a43f115749d419e3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 25 Aug 2022 12:16:10 +0100
Subject: [PATCH 383/394] [jsinterp] Fix bug in operator precedence * from
 https://github.com/yt-dlp/yt-dlp/commit/164b03c4864b0d44cfee5e7702f7c2317164a6cf
 * added tests

---
 test/test_jsinterp.py          | 25 +++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/jsinterp.py         |  7 ++++++-
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 96786a84c04..0a97bdbc411 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -192,6 +192,31 @@ def test_try(self):
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
+    def test_catch(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
+    @unittest.expectedFailure
+    def test_finally(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_nested_try(self):
+        jsi = JSInterpreter('''
+        function x() {try {
+            try{throw 10} finally {throw 42} 
+            } catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
         function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 327d4c40d3c..4bb0a30b0f6 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -111,6 +111,10 @@
         'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
         'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
     ),
+    (
+        'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
+        '5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 6719d0dfd77..a8456ec1c78 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -5,6 +5,7 @@
 import math
 import operator
 import re
+from collections import Counter
 
 from .utils import (
     error_to_compat_str,
@@ -108,8 +109,8 @@ def _js_ternary(cndn, if_true=True, if_false=False):
 
 _COMP_OPERATORS = (
     ('===', operator.is_),
-    ('==', _js_eq_op(operator.eq)),
     ('!==', operator.is_not),
+    ('==', _js_eq_op(operator.eq)),
     ('!=', _js_eq_op(operator.ne)),
     ('<=', _js_comp_op(operator.le)),
     ('>=', _js_comp_op(operator.ge)),
@@ -241,7 +242,9 @@ def _regex_flags(cls, expr):
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
+        # collections.Counter() is ~10% slower
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        # counters = Counter()
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
         after_op, in_regex_char_group, skip_re = True, False, 0
@@ -442,6 +445,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             return ret, should_abort or should_return
 
         elif md.get('catch'):
+
             catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             if self._EXC_NAME in local_vars:
                 catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
@@ -450,6 +454,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                     return ret, True
 
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+
             return ret, should_abort or should_return
 
         elif md.get('for'):

From 86018749387c18690c3c2745690ec9f9783e2d66 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 08:17:54 +0100
Subject: [PATCH 384/394] [jsinterp] Improve try/catch/finally support

---
 test/test_jsinterp.py  | 14 ++++++-
 youtube_dl/jsinterp.py | 92 +++++++++++++++++++++++-------------------
 2 files changed, 63 insertions(+), 43 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 0a97bdbc411..fb4882d0039 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -74,6 +74,9 @@ def test_operators(self):
         jsi = JSInterpreter('function f(){return 0 ?? 42;}')
         self.assertEqual(jsi.call_function('f'), 0)
 
+        jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
+        self.assertFalse(jsi.call_function('f'))
+
     def test_array_access(self):
         jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
@@ -198,7 +201,6 @@ def test_catch(self):
         ''')
         self.assertEqual(jsi.call_function('x'), 5)
 
-    @unittest.expectedFailure
     def test_finally(self):
         jsi = JSInterpreter('''
         function x() { try{throw 10} finally {return 42} }
@@ -212,7 +214,7 @@ def test_finally(self):
     def test_nested_try(self):
         jsi = JSInterpreter('''
         function x() {try {
-            try{throw 10} finally {throw 42} 
+            try{throw 10} finally {throw 42}
             } catch(e){return 5} }
         ''')
         self.assertEqual(jsi.call_function('x'), 5)
@@ -229,6 +231,14 @@ def test_for_loop_break(self):
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
+    def test_for_loop_try(self):
+        jsi = JSInterpreter('''
+        function x() {
+            for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
+            return 42 }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_literal_list(self):
         jsi = JSInterpreter('''
         function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a8456ec1c78..08726e4785b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -5,7 +5,6 @@
 import math
 import operator
 import re
-from collections import Counter
 
 from .utils import (
     error_to_compat_str,
@@ -15,6 +14,7 @@
     unified_timestamp,
 )
 from .compat import (
+    compat_basestring,
     compat_collections_chain_map as ChainMap,
     compat_itertools_zip_longest as zip_longest,
     compat_str,
@@ -76,6 +76,10 @@ def _js_comp_op(op):
     def wrapped(a, b):
         if JS_Undefined in (a, b):
             return False
+        if isinstance(a, compat_basestring):
+            b = compat_str(b or 0)
+        elif isinstance(b, compat_basestring):
+            a = compat_str(a or 0)
         return op(a or 0, b or 0)
 
     return wrapped
@@ -195,7 +199,6 @@ class JSInterpreter(object):
         'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
     }
 
-    _EXC_NAME = '__youtube_dl_exception__'
     _OBJ_NAME = '__youtube_dl_jsinterp_obj'
 
     OP_CHARS = None
@@ -242,9 +245,8 @@ def _regex_flags(cls, expr):
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
-        # collections.Counter() is ~10% slower
+        # collections.Counter() is ~10% slower in both 2.7 and 3.9
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        # counters = Counter()
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
         after_op, in_regex_char_group, skip_re = True, False, 0
@@ -291,7 +293,9 @@ def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         yield expr[start:]
 
     @classmethod
-    def _separate_at_paren(cls, expr, delim):
+    def _separate_at_paren(cls, expr, delim=None):
+        if delim is None:
+            delim = expr and _MATCHING_PARENS[expr[0]]
         separated = list(cls._separate(expr, delim, 1))
 
         if len(separated) < 2:
@@ -376,7 +380,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if expr.startswith('new '):
             obj = expr[4:]
             if obj.startswith('Date('):
-                left, right = self._separate_at_paren(obj[4:], ')')
+                left, right = self._separate_at_paren(obj[4:])
                 expr = unified_timestamp(
                     self.interpret_expression(left, local_vars, allow_recursion), False)
                 if not expr:
@@ -390,7 +394,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             return None, should_return
 
         if expr.startswith('{'):
-            inner, outer = self._separate_at_paren(expr, '}')
+            inner, outer = self._separate_at_paren(expr)
             # try for object expression (Map)
             sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
             if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
@@ -406,7 +410,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
-            inner, outer = self._separate_at_paren(expr, ')')
+            inner, outer = self._separate_at_paren(expr)
             inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
                 return inner, should_abort or should_return
@@ -414,57 +418,63 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('['):
-            inner, outer = self._separate_at_paren(expr, ']')
+            inner, outer = self._separate_at_paren(expr)
             name = self._named_object(local_vars, [
                 self.interpret_expression(item, local_vars, allow_recursion)
                 for item in self._separate(inner)])
             expr = name + outer
 
         m = re.match(r'''(?x)
-            (?P<try>try|finally)\s*|
-            (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
-            (?P<switch>switch)\s*\(|
-            (?P<for>for)\s*\(|
-            '''.format(**globals()), expr)
+                (?P<try>try)\s*\{|
+                (?P<switch>switch)\s*\(|
+                (?P<for>for)\s*\(
+                ''', expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
-            if expr[m.end()] == '{':
-                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
-            else:
-                try_expr, expr = expr[m.end() - 1:], ''
+            try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+            err = None
             try:
                 ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
                 if should_abort:
                     return ret, True
-            except JS_Throw as e:
-                local_vars[self._EXC_NAME] = e.error
             except Exception as e:
                 # XXX: This works for now, but makes debugging future issues very hard
-                local_vars[self._EXC_NAME] = e
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
-            return ret, should_abort or should_return
-
-        elif md.get('catch'):
-
-            catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
-            if self._EXC_NAME in local_vars:
-                catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
-                ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
+                err = e
+
+            pending = (None, False)
+            m = re.match(r'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{'.format(**globals()), expr)
+            if m:
+                sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                if err:
+                    catch_vars = {}
+                    if m.group('err'):
+                        catch_vars[m.group('err')] = err.error if isinstance(err, JS_Throw) else err
+                    catch_vars = local_vars.new_child(m=catch_vars)
+                    err = None
+                    pending = self.interpret_statement(sub_expr, catch_vars, allow_recursion)
+
+            m = re.match(r'finally\s*\{', expr)
+            if m:
+                sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
                 if should_abort:
                     return ret, True
 
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            ret, should_abort = pending
+            if should_abort:
+                return ret, True
 
-            return ret, should_abort or should_return
+            if err:
+                raise err
 
         elif md.get('for'):
-            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:])
             if remaining.startswith('{'):
-                body, expr = self._separate_at_paren(remaining, '}')
+                body, expr = self._separate_at_paren(remaining)
             else:
                 switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
                 if switch_m:
-                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
+                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:])
                     body, expr = self._separate_at_paren(remaining, '}')
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
@@ -483,11 +493,9 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 except JS_Continue:
                     pass
                 self.interpret_expression(increment, local_vars, allow_recursion)
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
-            return ret, should_abort or should_return
 
         elif md.get('switch'):
-            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:])
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
             items = body.replace('default:', 'case default:').split('case ')[1:]
@@ -510,6 +518,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                         break
                 if matched:
                     break
+
+        if md:
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
@@ -618,7 +628,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
                 member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
             if arg_str.startswith('('):
-                arg_str, remaining = self._separate_at_paren(arg_str, ')')
+                arg_str, remaining = self._separate_at_paren(arg_str)
             else:
                 arg_str, remaining = None, arg_str
 
@@ -795,7 +805,7 @@ def extract_function_code(self, funcname):
                 \((?P<args>[^)]*)\)\s*
                 (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
-        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
+        code, _ = self._separate_at_paren(func_m.group('code'))  # refine the match
         if func_m is None:
             raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
         return self.build_arglist(func_m.group('args')), code
@@ -810,7 +820,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             if mobj is None:
                 break
             start, body_start = mobj.span()
-            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            body, remaining = self._separate_at_paren(code[body_start - 1:])
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(

From cdfc866e6453c5d437335fd19e70997e5eacfe4b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 10:17:56 +0100
Subject: [PATCH 385/394] [compat] Replace deficient ChainMap class in Py3.3
 and earlier

---
 youtube_dl/compat.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 3002109ca52..366a93924a8 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3004,8 +3004,11 @@ def unpack(self, string):
 # new class in collections
 try:
     from collections import ChainMap as compat_collections_chain_map
+    # Py3.3's ChainMap is deficient
+    if sys.version_info <= (3, 3):
+        raise ImportError
 except ImportError:
-    # Py < 3.3
+    # Py <= 3.3
     class compat_collections_chain_map(compat_collections_abc.MutableMapping):
 
         maps = [{}]
@@ -3060,6 +3063,7 @@ def new_child(self, m=None, **kwargs):
         def parents(self):
             return compat_collections_chain_map(*(self.maps[1:]))
 
+
 # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
 compat_re_Pattern = type(re.compile(''))
 

From 7b4e2290bebddb4e2504596c92030e9eae93043e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 12:22:01 +0100
Subject: [PATCH 386/394] [compat] Replace deficient ChainMap class in Py3.3
 and earlier * fix version check

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 366a93924a8..eca6d63de72 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3005,7 +3005,7 @@ def unpack(self, string):
 try:
     from collections import ChainMap as compat_collections_chain_map
     # Py3.3's ChainMap is deficient
-    if sys.version_info <= (3, 3):
+    if sys.version_info < (3, 4):
         raise ImportError
 except ImportError:
     # Py <= 3.3

From bd41d33fc564d1d53000253e1ff0ae2fa954ff38 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 29 Aug 2022 13:02:17 +0100
Subject: [PATCH 387/394] [options] Document that postprocessing is not forced
 by --postprocessor-args

Resolves #30307
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f6621ef9160..f6d2b089889 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -801,7 +801,7 @@ def _comma_separated_values_options_callback(option, opt_str, value, parser):
     postproc.add_option(
         '--postprocessor-args',
         dest='postprocessor_args', metavar='ARGS',
-        help='Give these arguments to the postprocessor')
+        help='Give these arguments to the postprocessor (if postprocessing is required)')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,

From b35aee2f97379113efcc8fc50f07af0c122594ad Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 31 Aug 2022 23:22:48 +0100
Subject: [PATCH 388/394] [jsinterp] Handle new YT players 113ca41c, c57c113c *
 add NaN * allow any white-space character for `after_op` * align with yt-dlp
 f26af78a8ac11d9d617ed31ea5282cfaa5bcbcfa (charcodeAt and bitwise overflow) *
 allow escaping in regex, fixing player c57c113c

---
 test/test_jsinterp.py          | 21 ++++++++++++++++
 test/test_youtube_signature.py | 16 ++++++++++++
 youtube_dl/jsinterp.py         | 46 +++++++++++++++++++++-------------
 3 files changed, 65 insertions(+), 18 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index fb4882d0039..5121c8cf8da 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -135,6 +135,11 @@ def test_precedence(self):
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
     def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return NaN }
+        ''')
+        self.assertTrue(math.isnan(jsi.call_function('x')))
+
         jsi = JSInterpreter('''
         function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
         ''')
@@ -385,6 +390,22 @@ def test_regex(self):
         ''')
         self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
+    def test_char_code_at(self):
+        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
+        self.assertEqual(jsi.call_function('x', 0), 116)
+        self.assertEqual(jsi.call_function('x', 1), 101)
+        self.assertEqual(jsi.call_function('x', 2), 115)
+        self.assertEqual(jsi.call_function('x', 3), 116)
+        self.assertEqual(jsi.call_function('x', 4), None)
+        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+
+    def test_bitwise_operators_overflow(self):
+        jsi = JSInterpreter('function x(){return -524999584 << 5}')
+        self.assertEqual(jsi.call_function('x'), 379882496)
+
+        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
+        self.assertEqual(jsi.call_function('x'), 915423904)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4bb0a30b0f6..ec914a87133 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -111,10 +111,26 @@
         'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
         'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
     ),
+    (
+        'https://www.youtube.com/s/player/009f1d77/player_ias.vflset/en_US/base.js',
+        '5dwFHw8aFWQUQtffRq', 'audescmLUzI3jw',
+    ),
     (
         'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
         '5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
     ),
+    (
+        'https://www.youtube.com/s/player/c2199353/player_ias.vflset/en_US/base.js',
+        '5EHDMgYLV6HPGk_Mu-kk', 'AD5rgS85EkrE7',
+    ),
+    (
+        'https://www.youtube.com/s/player/113ca41c/player_ias.vflset/en_US/base.js',
+        'cgYl-tlYkhjT7A', 'hI7BBr2zUgcmMg',
+    ),
+    (
+        'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
+        '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 08726e4785b..d13329396f5 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -23,10 +23,11 @@
 
 def _js_bit_op(op):
 
+    def zeroise(x):
+        return 0 if x in (None, JS_Undefined) else x
+
     def wrapped(a, b):
-        def zeroise(x):
-            return 0 if x in (None, JS_Undefined) else x
-        return op(zeroise(a), zeroise(b))
+        return op(zeroise(a), zeroise(b)) & 0xffffffff
 
     return wrapped
 
@@ -44,7 +45,7 @@ def wrapped(a, b):
 def _js_div(a, b):
     if JS_Undefined in (a, b) or not (a and b):
         return float('nan')
-    return float('inf') if not b else operator.truediv(a or 0, b)
+    return operator.truediv(a or 0, b) if b else float('inf')
 
 
 def _js_mod(a, b):
@@ -260,13 +261,14 @@ def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
                     counters[_MATCHING_PARENS[char]] += 1
                 elif char in counters:
                     counters[char] -= 1
-            if not escaping and char in _QUOTES and in_quote in (char, None):
-                if in_quote or after_op or char != '/':
-                    in_quote = None if in_quote and not in_regex_char_group else char
-            elif in_quote == '/' and char in '[]':
-                in_regex_char_group = char == '['
+            if not escaping:
+                if char in _QUOTES and in_quote in (char, None):
+                    if in_quote or after_op or char != '/':
+                        in_quote = None if in_quote and not in_regex_char_group else char
+                elif in_quote == '/' and char in '[]':
+                    in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and char in cls.OP_CHARS or (char == ' ' and after_op)
+            after_op = not in_quote and (char in cls.OP_CHARS or (char.isspace() and after_op))
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -590,6 +592,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
 
         elif expr == 'undefined':
             return JS_Undefined, should_return
+        elif expr == 'NaN':
+            return float('NaN'), should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
@@ -635,7 +639,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             def assertion(cndn, msg):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
-                    raise ExtractorError('{member} {msg}'.format(**locals()), expr=expr)
+                    memb = member
+                    raise self.Exception('{member} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
                 if (variable, member) == ('console', 'debug'):
@@ -737,6 +742,13 @@ def eval_method():
                         return obj.index(idx, start)
                     except ValueError:
                         return -1
+                elif member == 'charCodeAt':
+                    assertion(isinstance(obj, compat_str), 'must be applied on a string')
+                    # assertion(len(argvals) == 1, 'takes exactly one argument') # but not enforced
+                    idx = argvals[0] if isinstance(argvals[0], int) else 0
+                    if idx >= len(obj):
+                        return None
+                    return ord(obj[idx])
 
                 idx = int(member) if isinstance(obj, list) else member
                 return obj[idx](argvals, allow_recursion=allow_recursion)
@@ -820,12 +832,10 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             if mobj is None:
                 break
             start, body_start = mobj.span()
-            body, remaining = self._separate_at_paren(code[body_start - 1:])
-            name = self._named_object(
-                local_vars,
-                self.extract_function_from_code(
-                    self.build_arglist(mobj.group('args')),
-                    body, local_vars, *global_stack))
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(local_vars, self.extract_function_from_code(
+                [x.strip() for x in mobj.group('args').split(',')],
+                body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)
 
@@ -854,7 +864,7 @@ def resf(args, kwargs={}, allow_recursion=100):
                 zip_longest(argnames, args, fillvalue=None))
             global_stack[0].update(kwargs)
             var_stack = LocalNameSpace(*global_stack)
-            ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)
+            ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1)
             if should_abort:
                 return ret
         return resf

From c342d50ce8b321437439ccb37db344c07c6c6e45 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 1 Sep 2022 13:28:30 +0100
Subject: [PATCH 389/394] [cache] Add cache validation by program version,
 based on yt-dlp

---
 test/test_cache.py  | 16 ++++++++++++++--
 youtube_dl/cache.py | 28 +++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/test/test_cache.py b/test/test_cache.py
index a161601420d..931074aa1de 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -3,17 +3,18 @@
 
 from __future__ import unicode_literals
 
-import shutil
-
 # Allow direct execution
 import os
 import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import shutil
 
 from test.helper import FakeYDL
 from youtube_dl.cache import Cache
+from youtube_dl.utils import version_tuple
+from youtube_dl.version import __version__
 
 
 def _is_empty(d):
@@ -54,6 +55,17 @@ def test_cache(self):
         self.assertFalse(os.path.exists(self.test_dir))
         self.assertEqual(c.load('test_cache', 'k.'), None)
 
+    def test_cache_validation(self):
+        ydl = FakeYDL({
+            'cachedir': self.test_dir,
+        })
+        c = Cache(ydl)
+        obj = {'x': 1, 'y': ['ä', '\\a', True]}
+        c.store('test_cache', 'k.', obj)
+        self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
+        new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
+        self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 7bdade1bdb4..4822439d0ed 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -10,12 +10,21 @@
 
 from .compat import compat_getenv
 from .utils import (
+    error_to_compat_str,
     expand_path,
+    is_outdated_version,
+    try_get,
     write_json_file,
 )
+from .version import __version__
 
 
 class Cache(object):
+
+    _YTDL_DIR = 'youtube-dl'
+    _VERSION_KEY = _YTDL_DIR + '_version'
+    _DEFAULT_VERSION = '2021.12.17'
+
     def __init__(self, ydl):
         self._ydl = ydl
 
@@ -23,7 +32,7 @@ def _get_root_dir(self):
         res = self._ydl.params.get('cachedir')
         if res is None:
             cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
-            res = os.path.join(cache_root, 'youtube-dl')
+            res = os.path.join(cache_root, self._YTDL_DIR)
         return expand_path(res)
 
     def _get_cache_fn(self, section, key, dtype):
@@ -50,13 +59,22 @@ def store(self, section, key, data, dtype='json'):
             except OSError as ose:
                 if ose.errno != errno.EEXIST:
                     raise
-            write_json_file(data, fn)
+            write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
         except Exception:
             tb = traceback.format_exc()
             self._ydl.report_warning(
                 'Writing cache to %r failed: %s' % (fn, tb))
 
-    def load(self, section, key, dtype='json', default=None):
+    def _validate(self, data, min_ver):
+        version = try_get(data, lambda x: x[self._VERSION_KEY])
+        if not version:  # Backward compatibility
+            data, version = {'data': data}, self._DEFAULT_VERSION
+        if not is_outdated_version(version, min_ver or '0', assume_new=False):
+            return data['data']
+        self._ydl.to_screen(
+            'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
+
+    def load(self, section, key, dtype='json', default=None, min_ver=None):
         assert dtype in ('json',)
 
         if not self.enabled:
@@ -66,12 +84,12 @@ def load(self, section, key, dtype='json', default=None):
         try:
             try:
                 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
-                    return json.load(cachef)
+                    return self._validate(json.load(cachef), min_ver)
             except ValueError:
                 try:
                     file_size = os.path.getsize(cache_fn)
                 except (OSError, IOError) as oe:
-                    file_size = str(oe)
+                    file_size = error_to_compat_str(oe)
                 self._ydl.report_warning(
                     'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
         except IOError:

From 5662dad552f3797c63d1ef980fca83f527e6c8c7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 2 Sep 2022 20:41:39 +0530
Subject: [PATCH 390/394] [jsinterp] Workaround operator associativity issue *
 temporary fix for player 5a3b6271 [1]

1. https://github.com/yt-dlp/yt-dlp/issues/4635#issuecomment-1235384480
---
 test/test_youtube_signature.py | 4 ++++
 youtube_dl/jsinterp.py         | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index ec914a87133..4e678cae00d 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -131,6 +131,10 @@
         'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
         '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
     ),
+    (
+        'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
+        'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index d13329396f5..99dd9843549 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -107,8 +107,8 @@ def _js_ternary(cndn, if_true=True, if_false=False):
     ('+', _js_arith_op(operator.add)),
     ('-', _js_arith_op(operator.sub)),
     ('*', _js_arith_op(operator.mul)),
-    ('/', _js_div),
     ('%', _js_mod),
+    ('/', _js_div),
     ('**', _js_exp),
 )
 

From f56d97bd74a102eeb5eafa2bdfb210357627ec49 Mon Sep 17 00:00:00 2001
From: Miguel Gargallo <mgargallo16@gmail.com>
Date: Tue, 4 Oct 2022 17:01:21 +0200
Subject: [PATCH 391/394] Fix Crucial guide command leads into errors

At university, colleagues and I tried this tool, but all of us got the error at this point:

  youtube-dl [OPTIONS] URL [URL...]

  youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best' URL https://www.youtube.com/watch?v=Toon9VARRIU

ERROR: You've asked youtube-dl to download the URL "URL". That doesn't make any sense. Simply remove the parameter in your command or configuration. Add -v to the command line to see what arguments and configuration youtube-dl got.

So, to avoid future devs into this error... we love the project and want to improve it!

By just removing the "URL" word.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8f0f01b227d..8163c26d367 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
 # DESCRIPTION
 **youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
 
-    youtube-dl [OPTIONS] URL [URL...]
+    youtube-dl [OPTIONS] [URL...]
 
 # OPTIONS
     -h, --help                           Print this help text and exit

From 3a6c11435c58fea465e4f1c51b109a5ff2cc23b6 Mon Sep 17 00:00:00 2001
From: Miguel Gargallo <mgargallo16@gmail.com>
Date: Thu, 6 Oct 2022 09:21:28 +0200
Subject: [PATCH 392/394] Improvement: Readme

Modified kine 52 to
youtube-dl [OPTIONS] URLS
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8163c26d367..c88b8be339f 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
 # DESCRIPTION
 **youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
 
-    youtube-dl [OPTIONS] [URL...]
+    youtube-dl [OPTIONS] URLS
 
 # OPTIONS
     -h, --help                           Print this help text and exit

From a602304244b04d18e312be9343e2b5824b6df8cc Mon Sep 17 00:00:00 2001
From: Miguel Gargallo <mgargallo16@gmail.com>
Date: Fri, 7 Oct 2022 15:18:47 +0200
Subject: [PATCH 393/394] Improvements

Improvements
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d928a34921b..9991bce717c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -131,7 +131,7 @@ def _list_from_options_callback(option, opt_str, value, parser, append=True, del
     kw = {
         'version': __version__,
         'formatter': fmt,
-        'usage': '%prog [OPTIONS] URL [URL...]',
+        'usage': '%prog [OPTIONS] URLS',
         'conflict_handler': 'resolve',
     }
 

From 2c0c42cd5c5689eed3dba0c16399c7b3386b44b9 Mon Sep 17 00:00:00 2001
From: GA <gaming-hacker@users.noreply.github.com>
Date: Sat, 8 Oct 2022 10:16:02 +0200
Subject: [PATCH 394/394] cleaning

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 03a9e5cc4c5..f2a2b43f3f5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2022.08.26'
+__version__ = '2022.10.08'