Fixed tests (+ Youtube shorts in channels)

extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -41,16 +41,12 @@ private NewPipe() {
     }
 
     public static void init(final Downloader d) {
-        downloader = d;
-        preferredLocalization = Localization.DEFAULT;
-        preferredContentCountry = ContentCountry.DEFAULT;
+        init(d, Localization.DEFAULT);
     }
 
     public static void init(final Downloader d, final Localization l) {
-        downloader = d;
-        preferredLocalization = l;
-        preferredContentCountry = l.getCountryCode().isEmpty()
-                ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode());
+        init(d, l, l.getCountryCode().isEmpty()
+                ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode()));
     }
 
     public static void init(final Downloader d, final Localization l, final ContentCountry c) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java

@@ -4,6 +4,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -14,10 +15,11 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.Comparator;
 
+import javax.annotation.Nonnull;
+
 public class MediaCCCRecentKiosk extends KioskExtractor<StreamInfoItem> {
 
     private JsonObject doc;
@@ -51,11 +53,17 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
                 streamInfoItem -> streamInfoItem.getUploadDate().offsetDateTime());
         comparator = comparator.reversed();
 
-        final StreamInfoItemsCollector collector
-                = new StreamInfoItemsCollector(getServiceId(), comparator);
-        for (int i = 0; i < events.size(); i++) {
-            collector.commit(new MediaCCCRecentKioskExtractor(events.getObject(i)));
-        }
+        final StreamInfoItemsCollector collector =
+                new StreamInfoItemsCollector(getServiceId(), comparator);
+
+        events.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(MediaCCCRecentKioskExtractor::new)
+                // #813 / voc/voctoweb#609 -> returns faulty data -> filter it out
+                .filter(extractor -> extractor.getDuration() > 0)
+                .forEach(collector::commit);
+
         return new InfoItemsPage<>(collector, null);
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java

@@ -1,16 +1,18 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
 import com.grack.nanojson.JsonObject;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
-import javax.annotation.Nullable;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 
+import javax.annotation.Nullable;
+
 public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
 
     private final JsonObject event;
@@ -45,7 +47,7 @@ public boolean isAd() {
     }
 
     @Override
-    public long getDuration() throws ParsingException {
+    public long getDuration() {
         // duration and length have the same value, see
         // https://github.com/voc/voctoweb/blob/master/app/views/public/shared/_event.json.jbuilder
         return event.getInt("duration");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -219,10 +219,34 @@ public static int parseDurationString(@Nonnull final String input)
                 throw new ParsingException("Error duration string with unknown format: " + input);
         }
 
-        return ((Integer.parseInt(Utils.removeNonDigitCharacters(days)) * 24
-                + Integer.parseInt(Utils.removeNonDigitCharacters(hours))) * 60
-                + Integer.parseInt(Utils.removeNonDigitCharacters(minutes))) * 60
-                + Integer.parseInt(Utils.removeNonDigitCharacters(seconds));
+        return ((convertDurationToInt(days) * 24
+                + convertDurationToInt(hours)) * 60
+                + convertDurationToInt(minutes)) * 60
+                + convertDurationToInt(seconds);
+    }
+
+    /**
+     * Tries to convert a duration string to an integer without throwing an exception.
+     * <br/>
+     * Helper method for {@link #parseDurationString(String)}.
+     * <br/>
+     * Note: This method is also used as a workaround for NewPipe#8034 (YT shorts no longer
+     * display any duration in channels).
+     *
+     * @param input The string to process
+     * @return The converted integer or 0 if the conversion failed.
+     */
+    private static int convertDurationToInt(final String input) {
+        if (input == null || input.isEmpty()) {
+            return 0;
+        }
+
+        final String clearedInput = Utils.removeNonDigitCharacters(input);
+        try {
+            return Integer.parseInt(clearedInput);
+        } catch (final NumberFormatException ex) {
+            return 0;
+        }
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -138,6 +138,11 @@ public long getDuration() throws ParsingException {
             }
         }
 
+        // NewPipe#8034 - YT returns not a correct duration for "YT shorts" videos
+        if ("SHORTS".equalsIgnoreCase(duration)) {
+            return 0;
+        }
+
         return YoutubeParsingHelper.parseDurationString(duration);
     }
 

extractor/src/test/java/org/schabi/newpipe/downloader/MockOnly.java

@@ -17,4 +17,9 @@
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(MockOnlyCondition.class)
 public @interface MockOnly {
+
+    /**
+     * The reason why the test is mockonly.
+     */
+    String value();
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java

@@ -1,17 +1,21 @@
 package org.schabi.newpipe.extractor.services.media_ccc;
 
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater;
+import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import java.util.stream.Stream;
 
 public class MediaCCCRecentListExtractorTest {
     private static KioskExtractor extractor;
@@ -24,16 +28,23 @@ public static void setUpClass() throws Exception {
     }
 
     @Test
-    public void testStreamList() throws Exception {
+    void testStreamList() throws Exception {
         final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
-        assertEquals(100, items.size());
-        for (final StreamInfoItem item: items) {
-            assertFalse(isNullOrEmpty(item.getName()));
-            assertTrue(item.getDuration() > 0);
-            // Disabled for now, because sometimes videos are uploaded, but their release date is in the future
-            // assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
-        }
-    }
+        assertFalse(items.isEmpty(), "No items returned");
 
+        assertAll(items.stream().flatMap(this::getAllConditionsForItem));
+    }
 
+    private Stream<Executable> getAllConditionsForItem(final StreamInfoItem item) {
+        return Stream.of(
+                () -> assertFalse(
+                        isNullOrEmpty(item.getName()),
+                        "Name=[" + item.getName() + "] of " + item + " is empty or null"
+                ),
+                () -> assertGreater(0,
+                        item.getDuration(),
+                        "Duration[=" + item.getDuration() + "] of " + item + " is <= 0"
+                )
+        );
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java

@@ -28,42 +28,47 @@ public static class Default {
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (PeertubeCommentsExtractor) PeerTube
-                    .getCommentsExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
+                    .getCommentsExtractor("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");
         }
 
         @Test
-        public void testGetComments() throws IOException, ExtractionException {
+        void testGetComments() throws IOException, ExtractionException {
+            final String comment = "I love this";
+
             InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
-            boolean result = findInComments(comments, "Cool.");
+            boolean result = findInComments(comments, comment);
 
             while (comments.hasNextPage() && !result) {
                 comments = extractor.getPage(comments.getNextPage());
-                result = findInComments(comments, "Cool.");
+                result = findInComments(comments, comment);
             }
 
             assertTrue(result);
         }
 
         @Test
-        public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
-            CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
+        void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
+            final String comment = "great video";
+
+            final CommentsInfo commentsInfo =
+                    CommentsInfo.getInfo("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");
             assertEquals("Comments", commentsInfo.getName());
 
-            boolean result = findInComments(commentsInfo.getRelatedItems(), "Cool");
+            boolean result = findInComments(commentsInfo.getRelatedItems(), comment);
 
             Page nextPage = commentsInfo.getNextPage();
             InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
             while (moreItems.hasNextPage() && !result) {
                 moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
-                result = findInComments(moreItems.getItems(), "Cool");
+                result = findInComments(moreItems.getItems(), comment);
                 nextPage = moreItems.getNextPage();
             }
 
             assertTrue(result);
         }
 
         @Test
-        public void testGetCommentsAllData() throws IOException, ExtractionException {
+        void testGetCommentsAllData() throws IOException, ExtractionException {
             InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
             for (CommentsInfoItem c : comments.getItems()) {
                 assertFalse(Utils.isBlank(c.getUploaderUrl()));
@@ -105,13 +110,13 @@ public static void setUp() throws Exception {
         }
 
         @Test
-        public void testGetComments() throws IOException, ExtractionException {
+        void testGetComments() throws IOException, ExtractionException {
             final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
             assertTrue(comments.getErrors().isEmpty());
         }
 
         @Test
-        public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
+        void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
             final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
             assertTrue(commentsInfo.getErrors().isEmpty());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java

@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
 
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -24,7 +25,7 @@ public static void setUp() {
     }
 
     @Test
-    public void acceptUrlTest() throws ParsingException {
+    void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909/videos"));
@@ -35,7 +36,7 @@ public void acceptUrlTest() throws ParsingException {
     }
 
     @Test
-    public void getIdFromUrl() throws ParsingException {
+    void getIdFromUrl() throws ParsingException {
         assertEquals("d8ca79f9-e4c7-4269-8183-d78ed269c909", linkHandler.getId("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
         assertEquals("d8ca79f9-e4c7-4269-8183-d78ed269c909", linkHandler.getId("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
         assertEquals("dacdc4ef-5160-4846-9b70-a655880da667", linkHandler.getId("https://framatube.org/videos/watch/playlist/dacdc4ef-5160-4846-9b70-a655880da667"));
@@ -47,9 +48,9 @@ public void getIdFromUrl() throws ParsingException {
     }
 
     @Test
-    public void getUrl() throws ParsingException {
-        System.out.println(linkHandler.fromUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());;
-        System.out.println(linkHandler.fromUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());;
-        System.out.println(linkHandler.fromUrl("https://framatube.org/w/p/sLFbqXsw7sPR3AfvqQSBZB").getUrl());;
+    void getUrl() {
+        assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());
+        assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());
+        assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/w/p/sLFbqXsw7sPR3AfvqQSBZB").getUrl());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -86,7 +86,7 @@ public void testGetLanguageInformation() throws ParsingException {
         @Override public long expectedViewCountAtLeast() { return 38600; }
         @Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
         @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
-        @Override public long expectedLikeCountAtLeast() { return 120; }
+        @Override public long expectedLikeCountAtLeast() { return 50; }
         @Override public long expectedDislikeCountAtLeast() { return 0; }
         @Override public String expectedHost() { return "framatube.org"; }
         @Override public String expectedCategory() { return "Science & Technology"; }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -399,7 +399,8 @@ public void testMoreRelatedItems() throws Exception {
 
         @Test
         public void testDescription() throws Exception {
-            ExtractorAsserts.assertContains("small team who want to make science look beautiful", extractor.getDescription());
+            ExtractorAsserts.assertContains("science", extractor.getDescription());
+            ExtractorAsserts.assertContains("animators", extractor.getDescription());
             //TODO: Description get cuts out, because the og:description is optimized and don't have all the content
             //assertTrue(description, description.contains("Currently we make one animation video per month"));
         }