Add support for the new channel layout (part 2)

locales/en-US.json

@@ -404,9 +404,7 @@
     "`x` marked it with a ❤": "`x` marked it with a ❤",
     "Audio mode": "Audio mode",
     "Video mode": "Video mode",
-    "Videos": "Videos",
     "Playlists": "Playlists",
-    "Community": "Community",
     "search_filters_title": "Filters",
     "search_filters_date_label": "Upload date",
     "search_filters_date_option_none": "Any date",
@@ -472,5 +470,11 @@
     "crash_page_read_the_faq": "read the <a href=\"`x`\">Frequently Asked Questions (FAQ)</a>",
     "crash_page_search_issue": "searched for <a href=\"`x`\">existing issues on GitHub</a>",
     "crash_page_report_issue": "If none of the above helped, please <a href=\"`x`\">open a new issue on GitHub</a> (preferably in English) and include the following text in your message (do NOT translate that text):",
-    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>"
+    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>",
+    "channel_tab_videos_label": "Videos",
+    "channel_tab_shorts_label": "Shorts",
+    "channel_tab_streams_label": "Livestreams",
+    "channel_tab_playlists_label": "Playlists",
+    "channel_tab_community_label": "Community",
+    "channel_tab_channels_label": "Channels"
 }

shard.lock

@@ -34,7 +34,7 @@ shards:
 
   protodec:
     git: https://github.com/iv-org/protodec.git
-    version: 0.1.4
+    version: 0.1.5
 
   radix:
     git: https://github.com/luislavena/radix.git

shard.yml

@@ -24,7 +24,7 @@ dependencies:
     version: ~> 0.6.1
   protodec:
     github: iv-org/protodec
-    version: ~> 0.1.4
+    version: ~> 0.1.5
   lsquic:
     github: iv-org/lsquic.cr
     version: ~> 2.18.1-2

spec/invidious/hashtag_spec.cr

@@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 1)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page1")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)
@@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 2)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page2")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)

spec/invidious/helpers_spec.cr

@@ -23,12 +23,6 @@ Spectator.describe "Helper" do
     end
   end
 
-  describe "#produce_channel_playlists_url" do
-    it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
-      expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
-    end
-  end
-
   describe "#produce_comment_continuation" do
     it "correctly produces a continuation token for comments" do
       expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D")

src/invidious.cr

@@ -48,6 +48,13 @@ require "./invidious/search/*"
 require "./invidious/routes/**"
 require "./invidious/jobs/**"
 
+# Declare the base namespace for invidious
+module Invidious
+end
+
+# Simple alias to make code easier to read
+alias IV = Invidious
+
 CONFIG   = Config.load
 HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
 
@@ -172,7 +179,7 @@ if CONFIG.popular_enabled
   Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
 end
 
-CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
+CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
 Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
 
 Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new

src/invidious/channels/about.cr

@@ -16,12 +16,6 @@ record AboutChannel,
   tabs : Array(String),
   verified : Bool
 
-record AboutRelatedChannel,
-  ucid : String,
-  author : String,
-  author_url : String,
-  author_thumbnail : String
-
 def get_about_info(ucid, locale) : AboutChannel
   begin
     # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@@ -100,34 +94,46 @@ def get_about_info(ucid, locale) : AboutChannel
   total_views = 0_i64
   joined = Time.unix(0)
 
-  tabs = [] of String
-
-  tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
-  if !tabs_json.nil?
-    # Retrieve information from the tabs array. The index we are looking for varies between channels.
-    tabs_json.each do |node|
-      # Try to find the about section which is located in only one of the tabs.
-      channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
-        .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
-          .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
-
-      if !channel_about_meta.nil?
-        total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
-
-        # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
-        joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
-          .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
-
-        # Normal Auto-generated channels
-        # https://support.google.com/youtube/answer/2579942
-        # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
-        if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
-           (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
-          auto_generated = true
-        end
-      end
+  tab_names = [] of String
+
+  if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
+    # Get the name of the tabs available on this channel
+    tab_names = tabs_json.as_a.compact_map do |entry|
+      name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
+
+      # This is a small fix to not add extra code on the HTML side
+      # I.e, the URL for the "live" tab is .../streams, so use "streams"
+      # everywhere for the sake of simplicity
+      (name == "live") ? "streams" : name
+    end
+
+    # Get the currently active tab ("About")
+    about_tab = extract_selected_tab(tabs_json)
+
+    # Try to find the about metadata section
+    channel_about_meta = about_tab.dig?(
+      "content",
+      "sectionListRenderer", "contents", 0,
+      "itemSectionRenderer", "contents", 0,
+      "channelAboutFullMetadataRenderer"
+    )
+
+    if !channel_about_meta.nil?
+      total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+
+      # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+      joined = extract_text(channel_about_meta["joinedDateText"]?)
+        .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
+      # Normal Auto-generated channels
+      # https://support.google.com/youtube/answer/2579942
+      # For auto-generated channels, channel_about_meta only has
+      # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+      auto_generated = (
+        (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
+           extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
+      )
     end
-    tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
   end
 
   sub_count = initdata
@@ -148,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel
     joined: joined,
     is_family_friendly: is_family_friendly,
     allowed_regions: allowed_regions,
-    tabs: tabs,
+    tabs: tab_names,
     verified: author_verified || false,
   )
 end
 
-def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
-  # params is {"2:string":"channels"} encoded
-  channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
-
-  tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
-  tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
-
-  return [] of AboutRelatedChannel if tab.nil?
-
-  items = tab.dig?(
-    "tabRenderer", "content",
-    "sectionListRenderer", "contents", 0,
-    "itemSectionRenderer", "contents", 0,
-    "gridRenderer", "items"
-  ).try &.as_a?
-
-  related = [] of AboutRelatedChannel
-  return related if (items.nil? || items.empty?)
-
-  items.each do |item|
-    renderer = item["gridChannelRenderer"]?
-    next if !renderer
-
-    related_id = renderer.dig("channelId").as_s
-    related_title = renderer.dig("title", "simpleText").as_s
-    related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
-    related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
-
-    related << AboutRelatedChannel.new(
-      ucid: related_id,
-      author: related_title,
-      author_url: related_author_url,
-      author_thumbnail: related_author_thumbnail,
-    )
+def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
+  if continuation.nil?
+    # params is {"2:string":"channels"} encoded
+    initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
+  else
+    initial_data = YoutubeAPI.browse(continuation)
   end
 
-  return related
+  items, continuation = extract_items(initial_data)
+
+  return items.select(SearchChannel), continuation
 end

src/invidious/channels/channels.cr

@@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
 
   LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
 
-  page = 1
+  channel = InvidiousChannel.new({
+    id:         ucid,
+    author:     author,
+    updated:    Time.utc,
+    deleted:    false,
+    subscribed: nil,
+  })
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
-  initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-  videos = extract_videos(initial_data, author, ucid)
+  videos, continuation = IV::Channel::Tabs.get_videos(channel)
 
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//feed/entry").each do |entry|
@@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
     views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
     views ||= 0_i64
 
-    channel_video = videos.select { |video| video.id == video_id }[0]?
+    channel_video = videos
+      .select(SearchVideo)
+      .select(&.id.== video_id)[0]?
 
     length_seconds = channel_video.try &.length_seconds
     length_seconds ||= 0
@@ -235,30 +242,25 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   end
 
   if pull_all_videos
-    page += 1
-
-    ids = [] of String
-
     loop do
-      initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-      videos = extract_videos(initial_data, author, ucid)
-
-      count = videos.size
-      videos = videos.map { |video| ChannelVideo.new({
-        id:                 video.id,
-        title:              video.title,
-        published:          video.published,
-        updated:            Time.utc,
-        ucid:               video.ucid,
-        author:             video.author,
-        length_seconds:     video.length_seconds,
-        live_now:           video.live_now,
-        premiere_timestamp: video.premiere_timestamp,
-        views:              video.views,
-      }) }
-
-      videos.each do |video|
-        ids << video.id
+      # Keep fetching videos using the continuation token retrieved earlier
+      videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
+
+      count = 0
+      videos.select(SearchVideo).each do |video|
+        count += 1
+        video = ChannelVideo.new({
+          id:                 video.id,
+          title:              video.title,
+          published:          video.published,
+          updated:            Time.utc,
+          ucid:               video.ucid,
+          author:             video.author,
+          length_seconds:     video.length_seconds,
+          live_now:           video.live_now,
+          premiere_timestamp: video.premiere_timestamp,
+          views:              video.views,
+        })
 
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
         # so since they don't provide a published date here we can safely ignore them.
@@ -269,17 +271,10 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       end
 
       break if count < 25
-      page += 1
+      sleep 500.milliseconds
     end
   end
 
-  channel = InvidiousChannel.new({
-    id:         ucid,
-    author:     author,
-    updated:    Time.utc,
-    deleted:    false,
-    subscribed: nil,
-  })
-
+  channel.updated = Time.utc
   return channel
 end