From 5ab6cd74200678ced00c2b47f14ab69568317fe1 Mon Sep 17 00:00:00 2001 From: chunky programmer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Thu, 11 May 2023 00:00:22 -0400 Subject: [PATCH 1/6] Extract YouTube playlist description --- .../schabi/newpipe/extractor/playlist/PlaylistExtractor.java | 2 ++ .../bandcamp/extractors/BandcampPlaylistExtractor.java | 5 +++++ .../peertube/extractors/PeertubePlaylistExtractor.java | 5 +++++ .../soundcloud/extractors/SoundcloudPlaylistExtractor.java | 5 +++++ .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 5 +++++ .../youtube/extractors/YoutubePlaylistExtractor.java | 5 +++++ 6 files changed, 27 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index baaac6b33b..25562673a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -21,6 +21,8 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l public abstract long getStreamCount() throws ParsingException; + public abstract String getDescription() throws ParsingException; + @Nonnull public String getThumbnailUrl() throws ParsingException { return ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 4514577635..462a040e88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -108,6 +108,11 @@ public long getStreamCount() { return trackInfo.size(); } + @Override + public String getDescription() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 71648b1ecd..6e848a6a9b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -65,6 +65,11 @@ public long getStreamCount() { return playlistInfo.getLong("videosLength"); } + @Override + public String getDescription() throws ParsingException { + return ""; + } + @Nonnull @Override public String getSubChannelName() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 13f1f14003..5ecb71ec76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -118,6 +118,11 @@ public long getStreamCount() { return playlist.getLong("track_count"); } + @Override + public String getDescription() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index fb574e294d..223fcf6c48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -169,6 +169,11 @@ public long getStreamCount() { return ListExtractor.ITEM_COUNT_INFINITE; } + @Override + public String getDescription() throws ParsingException { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 38f1dabbb0..390e637f74 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -294,6 +294,11 @@ public long getStreamCount() throws ParsingException { return ITEM_COUNT_UNKNOWN; } + @Override + public String getDescription() throws ParsingException { + return getTextFromObject(getPlaylistInfo().getObject("description")); + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { From e147867d410a4dcf175f32bcd623aeb370a7dad9 Mon Sep 17 00:00:00 2001 From: chunky programmer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Thu, 11 May 2023 00:00:34 -0400 Subject: [PATCH 2/6] Add tests --- .../youtube/YoutubePlaylistExtractorTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index f7ec34ec5d..e23b2f2e69 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; @@ -163,6 +164,12 @@ public void testUploaderVerified() throws Exception { void getPlaylistType() throws ParsingException { assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType()); } + + @Test + public void testDescription() throws ParsingException { + final String description = extractor.getDescription(); + assertContains("pop songs list", description); + } } public static class HugePlaylist implements BasePlaylistExtractorTest { @@ -286,6 +293,12 @@ public void testUploaderVerified() throws Exception { void getPlaylistType() throws ParsingException { assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType()); } + + @Test + public void testDescription() throws ParsingException { + final String description = extractor.getDescription(); + assertContains("I Wanna Rock Super Gigantic Playlist", description); + } } public static class LearningPlaylist implements BasePlaylistExtractorTest { @@ -394,6 +407,12 @@ public void testUploaderVerified() throws Exception { void getPlaylistType() throws ParsingException { assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType()); } + + @Test + public void testDescription() throws ParsingException { + final String description = extractor.getDescription(); + assertContains("47 episodes", description); + } } public static class ContinuationsTests { From 81f29116bad1655873c4eea8020af56ff5cd6516 Mon Sep 17 00:00:00 2001 From: chunky programmer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Thu, 11 May 2023 00:36:57 -0400 Subject: [PATCH 3/6] switch from string to Description object --- .../extractor/playlist/PlaylistExtractor.java | 3 ++- .../extractors/BandcampPlaylistExtractor.java | 5 +++-- .../extractors/PeertubePlaylistExtractor.java | 5 +++-- .../extractors/SoundcloudPlaylistExtractor.java | 5 +++-- .../extractors/YoutubeMixPlaylistExtractor.java | 5 +++-- .../extractors/YoutubePlaylistExtractor.java | 10 ++++++++-- .../youtube/YoutubePlaylistExtractorTest.java | 13 +++++++------ 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 25562673a1..c60738c91d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; @@ -21,7 +22,7 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l public abstract long getStreamCount() throws ParsingException; - public abstract String getDescription() throws ParsingException; + public abstract Description getDescription() throws ParsingException; @Nonnull public String getThumbnailUrl() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 462a040e88..a99c9c48de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -109,8 +110,8 @@ public long getStreamCount() { } @Override - public String getDescription() throws ParsingException { - return ""; + public Description getDescription() throws ParsingException { + return Description.EMPTY_DESCRIPTION; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 6e848a6a9b..e20b396fb3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -66,8 +67,8 @@ public long getStreamCount() { } @Override - public String getDescription() throws ParsingException { - return ""; + public Description getDescription() throws ParsingException { + return Description.EMPTY_DESCRIPTION; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 5ecb71ec76..34469a7cce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -119,8 +120,8 @@ public long getStreamCount() { } @Override - public String getDescription() throws ParsingException { - return ""; + public Description getDescription() throws ParsingException { + return Description.EMPTY_DESCRIPTION; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 223fcf6c48..5b6b1eeca8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -170,8 +171,8 @@ public long getStreamCount() { } @Override - public String getDescription() throws ParsingException { - return ""; + public Description getDescription() throws ParsingException { + return Description.EMPTY_DESCRIPTION; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 390e637f74..1523fc923b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -26,6 +26,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -295,8 +296,13 @@ public long getStreamCount() throws ParsingException { } @Override - public String getDescription() throws ParsingException { - return getTextFromObject(getPlaylistInfo().getObject("description")); + public Description getDescription() throws ParsingException { + final String description = getTextFromObject( + getPlaylistInfo().getObject("description"), + true + ); + + return new Description(description, Description.HTML); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index e23b2f2e69..3be2f8ce59 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -26,6 +26,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; @@ -167,8 +168,8 @@ void getPlaylistType() throws ParsingException { @Test public void testDescription() throws ParsingException { - final String description = extractor.getDescription(); - assertContains("pop songs list", description); + final Description description = extractor.getDescription(); + assertContains("pop songs list", description.getContent()); } } @@ -296,8 +297,8 @@ void getPlaylistType() throws ParsingException { @Test public void testDescription() throws ParsingException { - final String description = extractor.getDescription(); - assertContains("I Wanna Rock Super Gigantic Playlist", description); + final Description description = extractor.getDescription(); + assertContains("I Wanna Rock Super Gigantic Playlist", description.getContent()); } } @@ -410,8 +411,8 @@ void getPlaylistType() throws ParsingException { @Test public void testDescription() throws ParsingException { - final String description = extractor.getDescription(); - assertContains("47 episodes", description); + final Description description = extractor.getDescription(); + assertContains("47 episodes", description.getContent()); } } From b218bf69bd9172481f2ae06ef834e5916e83780a Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 12 May 2023 00:44:10 +0200 Subject: [PATCH 4/6] Implement PlaylistInfo.getDescription() Implement PlaylistExtractor.getDescription() for PeerTube and SoundCloud. Anotate PlaylistExtractor.getDescription() as Nonnull --- .../extractor/playlist/PlaylistExtractor.java | 1 + .../newpipe/extractor/playlist/PlaylistInfo.java | 15 +++++++++++++++ .../extractors/BandcampPlaylistExtractor.java | 1 + .../extractors/PeertubePlaylistExtractor.java | 7 ++++++- .../extractors/SoundcloudPlaylistExtractor.java | 7 ++++++- .../extractors/YoutubeMixPlaylistExtractor.java | 1 + .../extractors/YoutubePlaylistExtractor.java | 1 + .../peertube/PeertubePlaylistExtractorTest.java | 5 +++++ 8 files changed, 36 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index c60738c91d..bc4eee467c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -22,6 +22,7 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l public abstract long getStreamCount() throws ParsingException; + @Nonnull public abstract Description getDescription() throws ParsingException; @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index aea33304c9..97a0d530d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; @@ -102,6 +103,11 @@ public static PlaylistInfo getInfo(final PlaylistExtractor extractor) } catch (final Exception e) { info.addError(e); } + try { + info.setDescription(extractor.getDescription()); + } catch (final Exception e) { + info.addError(e); + } try { info.setThumbnailUrl(extractor.getThumbnailUrl()); } catch (final Exception e) { @@ -174,6 +180,7 @@ public static PlaylistInfo getInfo(final PlaylistExtractor extractor) private String subChannelName; private String subChannelAvatarUrl; private long streamCount = 0; + private Description description; private PlaylistType playlistType; public String getThumbnailUrl() { @@ -248,6 +255,14 @@ public void setStreamCount(final long streamCount) { this.streamCount = streamCount; } + public Description getDescription() { + return description; + } + + public void setDescription(final Description description) { + this.description = description; + } + public PlaylistType getPlaylistType() { return playlistType; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index a99c9c48de..0a0d255075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -109,6 +109,7 @@ public long getStreamCount() { return trackInfo.size(); } + @Nonnull @Override public Description getDescription() throws ParsingException { return Description.EMPTY_DESCRIPTION; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index e20b396fb3..3d4ff9d199 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -66,9 +66,14 @@ public long getStreamCount() { return playlistInfo.getLong("videosLength"); } + @Nonnull @Override public Description getDescription() throws ParsingException { - return Description.EMPTY_DESCRIPTION; + final String description = playlistInfo.getString("description"); + if (isNullOrEmpty(description)) { + return Description.EMPTY_DESCRIPTION; + } + return new Description(description, Description.PLAIN_TEXT); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 34469a7cce..88f38b1e1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -119,9 +119,14 @@ public long getStreamCount() { return playlist.getLong("track_count"); } + @Nonnull @Override public Description getDescription() throws ParsingException { - return Description.EMPTY_DESCRIPTION; + final String description = playlist.getString("description"); + if (isNullOrEmpty(description)) { + return Description.EMPTY_DESCRIPTION; + } + return new Description(description, Description.PLAIN_TEXT); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 5b6b1eeca8..eefc10a94d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -170,6 +170,7 @@ public long getStreamCount() { return ListExtractor.ITEM_COUNT_INFINITE; } + @Nonnull @Override public Description getDescription() throws ParsingException { return Description.EMPTY_DESCRIPTION; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 1523fc923b..06f055f14b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -295,6 +295,7 @@ public long getStreamCount() throws ParsingException { return ITEM_COUNT_UNKNOWN; } + @Nonnull @Override public Description getDescription() throws ParsingException { final String description = getTextFromObject( diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java index 5bd47de4a9..37853305ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -60,6 +60,11 @@ void testGetStreamCount() { ExtractorAsserts.assertGreaterOrEqual(39, extractor.getStreamCount()); } + @Test + void testGetDescription() throws ParsingException { + ExtractorAsserts.assertContains("épisodes de Shocking", extractor.getDescription().getContent()); + } + @Test void testGetSubChannelUrl() { assertEquals("https://skeptikon.fr/video-channels/metadechoc_channel", extractor.getSubChannelUrl()); From ca0ce00753581360867590a298d2a6d6b0283afc Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 12 May 2023 01:43:48 +0200 Subject: [PATCH 5/6] Add PlaylistInfoItem.getDescription() and PlaylistInfoItemExtractor.getDescription() [PeerTube] Implement the corresponding extractor method. TODO: add tests --- .../extractor/playlist/PlaylistInfoItem.java | 10 ++++++++++ .../playlist/PlaylistInfoItemExtractor.java | 11 +++++++++++ .../playlist/PlaylistInfoItemsCollector.java | 5 +++++ .../PeertubePlaylistInfoItemExtractor.java | 13 +++++++++++++ 4 files changed, 39 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java index 852a1819b4..10ff126835 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nullable; @@ -13,6 +14,7 @@ public class PlaylistInfoItem extends InfoItem { * How many streams this playlist have */ private long streamCount = 0; + private Description description; private PlaylistInfo.PlaylistType playlistType; public PlaylistInfoItem(final int serviceId, final String url, final String name) { @@ -52,6 +54,14 @@ public void setStreamCount(final long streamCount) { this.streamCount = streamCount; } + public Description getDescription() { + return description; + } + + public void setDescription(final Description description) { + this.description = description; + } + public PlaylistInfo.PlaylistType getPlaylistType() { return playlistType; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java index ccb3c2c7d1..e9c8231069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java @@ -2,6 +2,7 @@ import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nonnull; @@ -31,6 +32,16 @@ public interface PlaylistInfoItemExtractor extends InfoItemExtractor { */ long getStreamCount() throws ParsingException; + /** + * Get the description of the playlist if there is any. + * Otherwise, an {@link Description#EMPTY_DESCRIPTION EMPTY_DESCRIPTION} is returned. + * @return the playlist's description + */ + @Nonnull + default Description getDescription() throws ParsingException { + return Description.EMPTY_DESCRIPTION; + } + /** * @return the type of this playlist, see {@link PlaylistInfo.PlaylistType} for a description * of types. If not overridden always returns {@link PlaylistInfo.PlaylistType#NORMAL}. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java index 3184c67dc9..4fe35e40e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java @@ -41,6 +41,11 @@ public PlaylistInfoItem extract(final PlaylistInfoItemExtractor extractor) } catch (final Exception e) { addError(e); } + try { + resultItem.setDescription(extractor.getDescription()); + } catch (final Exception e) { + addError(e); + } try { resultItem.setPlaylistType(extractor.getPlaylistType()); } catch (final Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java index 219113da7d..168872e159 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java @@ -4,9 +4,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { final JsonObject item; @@ -54,4 +57,14 @@ public boolean isUploaderVerified() throws ParsingException { public long getStreamCount() throws ParsingException { return item.getInt("videosLength"); } + + @Nonnull + @Override + public Description getDescription() throws ParsingException { + final String description = item.getString("description"); + if (isNullOrEmpty(description)) { + return Description.EMPTY_DESCRIPTION; + } + return new Description(description, Description.PLAIN_TEXT); + } } From c70bb83801ff4dee66822b3f89a3d412460b8727 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 15 May 2023 15:23:03 +0200 Subject: [PATCH 6/6] [Bandcamp] Implement PlaylistExtractor.getDescsription() --- .../extractors/BandcampPlaylistExtractor.java | 25 ++++++++++++++++++- .../BandcampPlaylistExtractorTest.java | 12 +++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 0a0d255075..aa6dd79683 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -11,6 +11,8 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -25,6 +27,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.util.Objects; import javax.annotation.Nonnull; @@ -112,7 +115,27 @@ public long getStreamCount() { @Nonnull @Override public Description getDescription() throws ParsingException { - return Description.EMPTY_DESCRIPTION; + final Element tInfo = document.getElementById("trackInfo"); + if (tInfo == null) { + throw new ParsingException("Could not find trackInfo in document"); + } + final Elements about = tInfo.getElementsByClass("tralbum-about"); + final Elements credits = tInfo.getElementsByClass("tralbum-credits"); + final Element license = document.getElementById("license"); + if (about.isEmpty() && credits.isEmpty() && license == null) { + return Description.EMPTY_DESCRIPTION; + } + final StringBuilder sb = new StringBuilder(); + if (!about.isEmpty()) { + sb.append(Objects.requireNonNull(about.first()).html()); + } + if (!credits.isEmpty()) { + sb.append(Objects.requireNonNull(credits.first()).html()); + } + if (license != null) { + sb.append(license.html()); + } + return new Description(sb.toString(), Description.HTML); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 923d1c647c..f6f0979786 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -13,12 +13,14 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; import java.util.List; import static org.junit.jupiter.api.Assertions.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** @@ -135,6 +137,16 @@ public void testStreamCount() throws ParsingException { assertEquals(5, extractor.getStreamCount()); } + @Test + public void testDescription() throws ParsingException { + final Description description = extractor.getDescription(); + assertNotEquals(Description.EMPTY_DESCRIPTION, description); + assertContains("Artwork by Shona Radcliffe", description.getContent()); // about + assertContains("All tracks written, produced and recorded by Mac Benson", + description.getContent()); // credits + assertContains("all rights reserved", description.getContent()); // license + } + @Override public void testUploaderVerified() throws Exception { assertFalse(extractor.isUploaderVerified());