From ea5b08db4cc32e063bbdd6ba9f05e98ba0e2d4ab Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sun, 6 Aug 2017 17:20:15 -0300 Subject: [PATCH] Refactor and improvements --- build.gradle | 3 + .../schabi/newpipe/extractor/Extractor.java | 78 ++++++--- .../org/schabi/newpipe/extractor/Info.java | 4 +- .../newpipe/extractor/InfoItemCollector.java | 8 +- .../newpipe/extractor/ListExtractor.java | 60 ++++++- .../schabi/newpipe/extractor/ListInfo.java | 9 ++ .../org/schabi/newpipe/extractor/NewPipe.java | 89 +++++----- .../schabi/newpipe/extractor/ServiceList.java | 37 ++++- .../newpipe/extractor/StreamingService.java | 51 +++--- .../extractor/SuggestionExtractor.java | 3 +- .../newpipe/extractor/UrlIdHandler.java | 3 - .../extractor/channel/ChannelExtractor.java | 13 +- .../extractor/channel/ChannelInfo.java | 43 ++++- .../extractor/channel/ChannelInfoItem.java | 2 +- .../channel/ChannelInfoItemCollector.java | 2 +- .../channel/ChannelInfoItemExtractor.java | 2 +- .../extractor/playlist/PlaylistExtractor.java | 15 +- .../extractor/playlist/PlaylistInfo.java | 47 ++++-- .../extractor/playlist/PlaylistInfoItem.java | 2 +- .../playlist/PlaylistInfoItemCollector.java | 3 +- .../playlist/PlaylistInfoItemExtractor.java | 2 +- .../search/InfoItemSearchCollector.java | 4 +- .../extractor/search/SearchEngine.java | 2 +- .../extractor/search/SearchResult.java | 8 +- .../SoundcloudChannelExtractor.java | 92 +++++------ .../SoundcloudChannelInfoItemExtractor.java | 2 +- .../SoundcloudChannelUrlIdHandler.java | 37 +---- .../soundcloud/SoundcloudParsingHelper.java | 145 ++++++++++++----- .../SoundcloudPlaylistExtractor.java | 92 +++++------ .../SoundcloudPlaylistUrlIdHandler.java | 32 +--- .../soundcloud/SoundcloudSearchEngine.java | 8 +- .../soundcloud/SoundcloudService.java | 47 ++---- .../soundcloud/SoundcloudStreamExtractor.java | 63 ++++---- .../SoundcloudStreamInfoItemExtractor.java | 2 +- .../SoundcloudStreamUrlIdHandler.java | 33 +--- .../SoundcloudSuggestionExtractor.java | 10 +- .../extractor/services/youtube/ItagItem.java | 2 +- .../youtube/YoutubeChannelExtractor.java | 123 ++++++-------- .../YoutubeChannelInfoItemExtractor.java | 2 +- .../youtube/YoutubePlaylistExtractor.java | 152 +++++++----------- .../services/youtube/YoutubeService.java | 47 ++---- .../youtube/YoutubeStreamExtractor.java | 84 ++++++---- .../YoutubeStreamInfoItemExtractor.java | 11 +- .../youtube/YoutubeSuggestionExtractor.java | 53 ++---- .../newpipe/extractor/stream/Stream.java | 2 +- .../extractor/stream/StreamExtractor.java | 25 +-- .../newpipe/extractor/stream/StreamInfo.java | 35 ++-- .../extractor/utils/DashMpdParser.java | 8 +- .../schabi/newpipe/extractor/utils/Utils.java | 18 ++- .../java/org/schabi/newpipe/Downloader.java | 56 ++++--- .../schabi/newpipe/extractor/NewPipeTest.java | 74 +++++++++ .../youtube/YoutubeChannelExtractorTest.java | 39 +++-- .../youtube/YoutubePlaylistExtractorTest.java | 98 +++++++++++ .../youtube/YoutubeSearchEngineAllTest.java | 9 +- .../YoutubeSearchEngineChannelTest.java | 15 +- .../YoutubeSearchEngineStreamTest.java | 18 ++- .../YoutubeStreamExtractorDefaultTest.java | 34 ++-- .../YoutubeStreamExtractorGemaTest.java | 53 ++++++ .../YoutubeStreamExtractorRestrictedTest.java | 108 +++++++++++++ .../YoutubeStreamUrlIdHandlerTest.java | 117 ++++++++++++++ .../YoutubeSuggestionExtractorTest.java | 51 ++++++ 61 files changed, 1444 insertions(+), 843 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/ListInfo.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorGemaTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java diff --git a/build.gradle b/build.gradle index 0bd985d150..17896966ba 100644 --- a/build.gradle +++ b/build.gradle @@ -10,5 +10,8 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' testImplementation 'junit:junit:4.12' + + sourceCompatibility = 1.7 + targetCompatibility = 1.7 } diff --git a/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/src/main/java/org/schabi/newpipe/extractor/Extractor.java index c704e6f56e..39a39ae05d 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,35 +1,73 @@ package org.schabi.newpipe.extractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; -import java.io.Serializable; +import java.io.IOException; -public abstract class Extractor implements Serializable { - private final int serviceId; - private final String url; - private final UrlIdHandler urlIdHandler; - private final StreamInfoItemCollector previewInfoCollector; +public abstract class Extractor { + /** + * {@link StreamingService} currently related to this extractor.
+ * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls). + */ + private final StreamingService service; - public Extractor(UrlIdHandler urlIdHandler, int serviceId, String url) { - this.urlIdHandler = urlIdHandler; - this.serviceId = serviceId; - this.url = url; - this.previewInfoCollector = new StreamInfoItemCollector(serviceId); + /** + * Dirty/original url that was passed in the constructor. + *

+ * What makes a url "dirty" or not is, for example, the additional parameters + * (not important as—in this case—the id): + * 

+     *     https://www.youtube.com/watch?v=a9Zf_258aTI&t=4s  →  &t=4s
+     *
+ * But as you can imagine, the time parameter is very important when calling, for example, {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}. + */ + private final String originalUrl; + + /** + * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}). + *

+ * Is lazily-cleaned by calling {@link #getCleanUrl()} + */ + private String cleanUrl; + + public Extractor(StreamingService service, String url) throws ExtractionException { + this.service = service; + this.originalUrl = url; } - public String getUrl() { - return url; + /** + * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler). + */ + protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException; + + /** + * Fetch the current page. + */ + public abstract void fetchPage() throws IOException, ExtractionException; + + public String getOriginalUrl() { + return originalUrl; } - public UrlIdHandler getUrlIdHandler() { - return urlIdHandler; + public String getCleanUrl() { + if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl; + + try { + cleanUrl = getUrlIdHandler().cleanUrl(originalUrl); + } catch (Exception e) { + cleanUrl = null; + // Fallback to the original url + return originalUrl; + } + return cleanUrl; } - public int getServiceId() { - return serviceId; + public StreamingService getService() { + return service; } - protected StreamInfoItemCollector getStreamPreviewInfoCollector() { - return previewInfoCollector; + public int getServiceId() { + return service.getServiceId(); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/Info.java b/src/main/java/org/schabi/newpipe/extractor/Info.java index f55a5740d7..6443b3d574 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; -import java.util.Vector; public abstract class Info implements Serializable { @@ -15,5 +15,5 @@ public abstract class Info implements Serializable { public String url; public String name; - public List errors = new Vector<>(); + public List errors = new ArrayList<>(); } diff --git a/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java index b55deda049..f65372224b 100644 --- a/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java @@ -2,8 +2,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import java.util.ArrayList; import java.util.List; -import java.util.Vector; /* * Created by Christian Schabesberger on 12.02.17. @@ -26,8 +26,8 @@ */ public abstract class InfoItemCollector { - private List itemList = new Vector<>(); - private List errors = new Vector<>(); + private List itemList = new ArrayList<>(); + private List errors = new ArrayList<>(); private int serviceId = -1; public InfoItemCollector(int serviceId) { @@ -46,7 +46,7 @@ protected void addFromCollector(InfoItemCollector otherC) throws ExtractionExcep if (serviceId != otherC.serviceId) { throw new ExtractionException("Service Id does not equal: " + NewPipe.getNameOfService(serviceId) - + " and " + NewPipe.getNameOfService(otherC.serviceId)); + + " and " + NewPipe.getNameOfService((otherC.serviceId))); } errors.addAll(otherC.errors); itemList.addAll(otherC.itemList); diff --git a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index 318cde7d5d..7af2f75e8e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; +import java.util.List; /** * Base class to extractors that have a list (e.g. playlists, channels). @@ -11,15 +12,39 @@ public abstract class ListExtractor extends Extractor { protected String nextStreamsUrl; - public ListExtractor(UrlIdHandler urlIdHandler, int serviceId, String url) { - super(urlIdHandler, serviceId, url); + /** + * Get a new ListExtractor with the given nextStreamsUrl set. + *

+ * The extractor WILL fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will NOT. + *

+ * You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth + * downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}. + */ + public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url); + setNextStreamsUrl(nextStreamsUrl); + + if (fetchPageUponCreation()) { + fetchPage(); + } } - public boolean hasMoreStreams(){ - return nextStreamsUrl != null && !nextStreamsUrl.isEmpty(); + /** + * Decide if the page will be fetched upon creation. + *

+ * The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller + * don't need or know what is the next page, thus, fetch the page). + */ + protected boolean fetchPageUponCreation() { + return nextStreamsUrl == null || nextStreamsUrl.isEmpty(); } - public abstract StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException; + public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException; + public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException; + + public boolean hasMoreStreams() { + return nextStreamsUrl != null && !nextStreamsUrl.isEmpty(); + } public String getNextStreamsUrl() { return nextStreamsUrl; @@ -29,4 +54,29 @@ public void setNextStreamsUrl(String nextStreamsUrl) { this.nextStreamsUrl = nextStreamsUrl; } + /*////////////////////////////////////////////////////////////////////////// + // Inner + //////////////////////////////////////////////////////////////////////////*/ + + public static class NextItemsResult { + /** + * The current list of items to this result + */ + public final List nextItemsList; + + /** + * Next url to fetch more items + */ + public final String nextItemsUrl; + + public NextItemsResult(List nextItemsList, String nextItemsUrl) { + this.nextItemsList = nextItemsList; + this.nextItemsUrl = nextItemsUrl; + } + + public boolean hasMoreStreams() { + return nextItemsUrl != null && !nextItemsUrl.isEmpty(); + } + } + } diff --git a/src/main/java/org/schabi/newpipe/extractor/ListInfo.java b/src/main/java/org/schabi/newpipe/extractor/ListInfo.java new file mode 100644 index 0000000000..d625c14e85 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/ListInfo.java @@ -0,0 +1,9 @@ +package org.schabi.newpipe.extractor; + +import java.util.List; + +public abstract class ListInfo extends Info { + public List related_streams; + public boolean has_more_streams; + public String next_streams_url; +} diff --git a/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 044c859671..7b4057d94a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -1,7 +1,5 @@ package org.schabi.newpipe.extractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - /* * Created by Christian Schabesberger on 23.08.15. * @@ -22,70 +20,83 @@ * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + /** - * Provides access to the video streaming services supported by NewPipe. - * Currently only Youtube until the API becomes more stable. + * Provides access to streaming services supported by NewPipe. */ - -@SuppressWarnings("ALL") public class NewPipe { private static final String TAG = NewPipe.class.toString(); + private static Downloader downloader = null; private NewPipe() { } - private static Downloader downloader = null; + public static void init(Downloader d) { + downloader = d; + } + + public static Downloader getDownloader() { + return downloader; + } + + /*////////////////////////////////////////////////////////////////////////// + // Utils + //////////////////////////////////////////////////////////////////////////*/ public static StreamingService[] getServices() { - return ServiceList.serviceList; + final ServiceList[] values = ServiceList.values(); + final StreamingService[] streamingServices = new StreamingService[values.length]; + + for (int i = 0; i < values.length; i++) streamingServices[i] = values[i].getService(); + + return streamingServices; } public static StreamingService getService(int serviceId) throws ExtractionException { - for (StreamingService s : ServiceList.serviceList) { - if (s.getServiceId() == serviceId) { - return s; + for (ServiceList item : ServiceList.values()) { + if (item.getService().getServiceId() == serviceId) { + return item.getService(); } } - return null; + throw new ExtractionException("There's no service with the id = \"" + serviceId + "\""); } public static StreamingService getService(String serviceName) throws ExtractionException { - return ServiceList.serviceList[getIdOfService(serviceName)]; - } - - public static String getNameOfService(int id) { - try { - return getService(id).getServiceInfo().name; - } catch (Exception e) { - System.err.println("Service id not known"); - e.printStackTrace(); - return ""; + for (ServiceList item : ServiceList.values()) { + if (item.getService().getServiceInfo().name.equals(serviceName)) { + return item.getService(); + } } + throw new ExtractionException("There's no service with the name = \"" + serviceName + "\""); } - public static int getIdOfService(String serviceName) { - for (int i = 0; i < ServiceList.serviceList.length; i++) { - if (ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) { - return i; + public static StreamingService getServiceByUrl(String url) throws ExtractionException { + for (ServiceList item : ServiceList.values()) { + if (item.getService().getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) { + return item.getService(); } } - return -1; + throw new ExtractionException("No service can handle the url = \"" + url + "\""); } - public static void init(Downloader d) { - downloader = d; - } - - public static Downloader getDownloader() { - return downloader; + public static int getIdOfService(String serviceName) { + try { + //noinspection ConstantConditions + return getService(serviceName).getServiceId(); + } catch (ExtractionException ignored) { + return -1; + } } - public static StreamingService getServiceByUrl(String url) { - for (StreamingService s : ServiceList.serviceList) { - if (s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) { - return s; - } + public static String getNameOfService(int id) { + try { + //noinspection ConstantConditions + return getService(id).getServiceInfo().name; + } catch (Exception e) { + System.err.println("Service id not known"); + e.printStackTrace(); + return ""; } - return null; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 26a7483684..64f79ff252 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,15 +1,36 @@ package org.schabi.newpipe.extractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; +import org.schabi.newpipe.extractor.services.youtube.YoutubeService; -/* - * Created by the-scrabi on 18.02.17. +/** + * A list of supported services. */ +public enum ServiceList { + Youtube(new YoutubeService(0, "Youtube")), + SoundCloud(new SoundcloudService(1, "SoundCloud")); +// DailyMotion(new DailyMotionService(2, "DailyMotion")); + + private final StreamingService service; + + ServiceList(StreamingService service) { + this.service = service; + } + + public StreamingService getService() { + return service; + } + + public StreamingService.ServiceInfo getServiceInfo() { + return service.getServiceInfo(); + } + + public int getId() { + return service.getServiceId(); + } -class ServiceList { - public static final StreamingService[] serviceList = { - new YoutubeService(0), - new SoundcloudService(1) - }; + @Override + public String toString() { + return service.getServiceInfo().name; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 8d5b87ffa6..ae2241f305 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -10,7 +10,11 @@ public abstract class StreamingService { public class ServiceInfo { - public String name = ""; + public final String name; + + public ServiceInfo(String name) { + this.name = name; + } } public enum LinkType { @@ -20,35 +24,46 @@ public enum LinkType { PLAYLIST } - private int serviceId; + private final int serviceId; + private final ServiceInfo serviceInfo; - public StreamingService(int id) { - serviceId = id; + public StreamingService(int id, String name) { + this.serviceId = id; + this.serviceInfo = new ServiceInfo(name); + } + + public final int getServiceId() { + return serviceId; } - public abstract ServiceInfo getServiceInfo(); + public ServiceInfo getServiceInfo() { + return serviceInfo; + } - public abstract UrlIdHandler getStreamUrlIdHandlerInstance(); - public abstract UrlIdHandler getChannelUrlIdHandlerInstance(); - public abstract UrlIdHandler getPlaylistUrlIdHandlerInstance(); - public abstract SearchEngine getSearchEngineInstance(); - public abstract SuggestionExtractor getSuggestionExtractorInstance(); - public abstract StreamExtractor getStreamExtractorInstance(String url) throws IOException, ExtractionException; - public abstract ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException; - public abstract PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException; + public abstract UrlIdHandler getStreamUrlIdHandler(); + public abstract UrlIdHandler getChannelUrlIdHandler(); + public abstract UrlIdHandler getPlaylistUrlIdHandler(); + public abstract SearchEngine getSearchEngine(); + public abstract SuggestionExtractor getSuggestionExtractor(); + public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; + public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; + public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; + public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { + return getChannelExtractor(url, null); + } - public final int getServiceId() { - return serviceId; + public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException { + return getPlaylistExtractor(url, null); } /** * figure out where the link is pointing to (a channel, video, playlist, etc.) */ public final LinkType getLinkTypeByUrl(String url) { - UrlIdHandler sH = getStreamUrlIdHandlerInstance(); - UrlIdHandler cH = getChannelUrlIdHandlerInstance(); - UrlIdHandler pH = getPlaylistUrlIdHandlerInstance(); + UrlIdHandler sH = getStreamUrlIdHandler(); + UrlIdHandler cH = getChannelUrlIdHandler(); + UrlIdHandler pH = getPlaylistUrlIdHandler(); if (sH.acceptUrl(url)) { return LinkType.STREAM; diff --git a/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java index 8750c73d84..983adc07e3 100644 --- a/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java @@ -33,8 +33,7 @@ public SuggestionExtractor(int serviceId) { this.serviceId = serviceId; } - public abstract List suggestionList(String query, String contentCountry) - throws ExtractionException, IOException; + public abstract List suggestionList(String query, String contentCountry) throws IOException, ExtractionException; public int getServiceId() { return serviceId; diff --git a/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java index 9e5739e334..84a80b9dac 100644 --- a/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java @@ -1,9 +1,6 @@ package org.schabi.newpipe.extractor; -import java.io.IOException; - import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; /* * Created by Christian Schabesberger on 26.07.16. diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 64311ef06f..bdcad4527a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,11 +1,10 @@ package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; @@ -31,8 +30,13 @@ public abstract class ChannelExtractor extends ListExtractor { - public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, serviceId, url); + public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + + @Override + protected UrlIdHandler getUrlIdHandler() throws ParsingException { + return getService().getChannelUrlIdHandler(); } public abstract String getChannelId() throws ParsingException; @@ -40,7 +44,6 @@ public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) th public abstract String getAvatarUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; - public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException; public abstract long getSubscriberCount() throws ParsingException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 6ccff7814c..47473de2d8 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -1,11 +1,16 @@ package org.schabi.newpipe.extractor.channel; -import org.schabi.newpipe.extractor.Info; -import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult; +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; -import java.util.List; +import java.io.IOException; +import java.util.ArrayList; /* * Created by Christian Schabesberger on 31.07.16. @@ -27,17 +32,36 @@ * along with NewPipe. If not, see . */ -public class ChannelInfo extends Info { +public class ChannelInfo extends ListInfo { + + public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException { + return getMoreItems(serviceItem.getService(), nextStreamsUrl); + } + + public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException { + return service.getChannelExtractor(null, nextStreamsUrl).getNextStreams(); + } + + public static ChannelInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static ChannelInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url); + } + + public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + return getInfo(service.getChannelExtractor(url)); + } public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException { ChannelInfo info = new ChannelInfo(); // important data info.service_id = extractor.getServiceId(); - info.url = extractor.getUrl(); + info.url = extractor.getCleanUrl(); info.id = extractor.getChannelId(); info.name = extractor.getChannelName(); - info.has_more_streams = extractor.hasMoreStreams(); try { info.avatar_url = extractor.getAvatarUrl(); @@ -67,13 +91,16 @@ public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingExce info.errors.add(e); } + // Lists can be null if a exception was thrown during extraction + if (info.related_streams == null) info.related_streams = new ArrayList<>(); + + info.has_more_streams = extractor.hasMoreStreams(); + info.next_streams_url = extractor.getNextStreamsUrl(); return info; } public String avatar_url; public String banner_url; public String feed_url; - public List related_streams; public long subscriber_count = -1; - public boolean has_more_streams = false; } diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java index 367c1183e2..e0ff5a4d30 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java @@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem { public String thumbnail_url; public String description; public long subscriber_count = -1; - public long view_count = -1; + public long stream_count = -1; public ChannelInfoItem() { super(InfoType.CHANNEL); diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java index feca36503d..8e8c45d2a4 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java @@ -43,7 +43,7 @@ public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws Parsin addError(e); } try { - resultItem.view_count = extractor.getViewCount(); + resultItem.stream_count = extractor.getStreamCount(); } catch (Exception e) { addError(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java index c8c08e641f..5f22ca7aa6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java @@ -28,5 +28,5 @@ public interface ChannelInfoItemExtractor { String getWebPageUrl() throws ParsingException; String getDescription() throws ParsingException; long getSubscriberCount() throws ParsingException; - long getViewCount() throws ParsingException; + long getStreamCount() throws ParsingException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 514c10ece1..f9f62e9cc5 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,18 +1,22 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; public abstract class PlaylistExtractor extends ListExtractor { - public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, serviceId, url); + public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + + @Override + protected UrlIdHandler getUrlIdHandler() throws ParsingException { + return getService().getPlaylistUrlIdHandler(); } public abstract String getPlaylistId() throws ParsingException; @@ -22,6 +26,5 @@ public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) t public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderName() throws ParsingException; public abstract String getUploaderAvatarUrl() throws ParsingException; - public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException; - public abstract long getStreamsCount() throws ParsingException; + public abstract long getStreamCount() throws ParsingException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 18cb8a5952..6e296dd765 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -1,25 +1,49 @@ package org.schabi.newpipe.extractor.playlist; -import org.schabi.newpipe.extractor.Info; -import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult; +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; -import java.util.List; +import java.io.IOException; +import java.util.ArrayList; -public class PlaylistInfo extends Info { +public class PlaylistInfo extends ListInfo { + + public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException { + return getMoreItems(serviceItem.getService(), nextStreamsUrl); + } + + public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException { + return service.getPlaylistExtractor(null, nextStreamsUrl).getNextStreams(); + } + + public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static PlaylistInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url); + } + + public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + return getInfo(service.getPlaylistExtractor(url)); + } public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException { PlaylistInfo info = new PlaylistInfo(); info.service_id = extractor.getServiceId(); - info.url = extractor.getUrl(); + info.url = extractor.getCleanUrl(); info.id = extractor.getPlaylistId(); info.name = extractor.getPlaylistName(); - info.has_more_streams = extractor.hasMoreStreams(); try { - info.streams_count = extractor.getStreamsCount(); + info.stream_count = extractor.getStreamCount(); } catch (Exception e) { info.errors.add(e); } @@ -56,6 +80,11 @@ public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingEx info.errors.add(e); } + // Lists can be null if a exception was thrown during extraction + if (info.related_streams == null) info.related_streams = new ArrayList<>(); + + info.has_more_streams = extractor.hasMoreStreams(); + info.next_streams_url = extractor.getNextStreamsUrl(); return info; } @@ -64,7 +93,5 @@ public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingEx public String uploader_url; public String uploader_name; public String uploader_avatar_url; - public long streams_count = 0; - public List related_streams; - public boolean has_more_streams; + public long stream_count = 0; } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java index 6a4eafcc23..d49521586c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java @@ -8,7 +8,7 @@ public class PlaylistInfoItem extends InfoItem { /** * How many streams this playlist have */ - public long streams_count = 0; + public long stream_count = 0; public PlaylistInfoItem() { super(InfoType.PLAYLIST); diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java index e3e764c4ed..6ad2e15095 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.InfoItemCollector; -import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; public class PlaylistInfoItemCollector extends InfoItemCollector { @@ -22,7 +21,7 @@ public PlaylistInfoItem extract(PlaylistInfoItemExtractor extractor) throws Pars addError(e); } try { - resultItem.streams_count = extractor.getStreamsCount(); + resultItem.stream_count = extractor.getStreamCount(); } catch (Exception e) { addError(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java index 66ea009ae7..a97a1a33f2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java @@ -6,5 +6,5 @@ public interface PlaylistInfoItemExtractor { String getThumbnailUrl() throws ParsingException; String getPlaylistName() throws ParsingException; String getWebPageUrl() throws ParsingException; - long getStreamsCount() throws ParsingException; + long getStreamCount() throws ParsingException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java index c33ddefe06..9dc647988c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java @@ -59,7 +59,7 @@ public void commit(StreamInfoItemExtractor extractor) { try { result.resultList.add(streamCollector.extract(extractor)); } catch (FoundAdException ae) { - System.err.println("Found add"); + System.err.println("Found ad"); } catch (Exception e) { addError(e); } @@ -69,7 +69,7 @@ public void commit(ChannelInfoItemExtractor extractor) { try { result.resultList.add(channelCollector.extract(extractor)); } catch (FoundAdException ae) { - System.err.println("Found add"); + System.err.println("Found ad"); } catch (Exception e) { addError(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java index d77812c7b8..83308136bc 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java @@ -49,5 +49,5 @@ protected InfoItemSearchCollector getInfoItemSearchCollector() { //Result search(String query, int page); public abstract InfoItemSearchCollector search( String query, int page, String contentCountry, EnumSet filter) - throws ExtractionException, IOException; + throws IOException, ExtractionException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java index 0783d3d6ae..11c9850ba2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java @@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import java.io.IOException; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; -import java.util.Vector; /* * Created by Christian Schabesberger on 29.02.16. @@ -31,7 +31,7 @@ public class SearchResult { public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, EnumSet filter) - throws ExtractionException, IOException { + throws IOException, ExtractionException { SearchResult result = engine .search(query, page, languageCode, filter) @@ -50,6 +50,6 @@ public static SearchResult getSearchResult(SearchEngine engine, String query, } public String suggestion; - public List resultList = new Vector<>(); - public List errors = new Vector<>(); + public List resultList = new ArrayList<>(); + public List errors = new ArrayList<>(); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 7c1f0be6e9..e68d312f20 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -1,37 +1,46 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; - -import org.json.JSONArray; import org.json.JSONObject; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; +import java.io.IOException; + @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { private String channelId; private JSONObject channel; - private String nextUrl; - public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, url, serviceId); + public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + @Override + public void fetchPage() throws IOException, ExtractionException { Downloader dl = NewPipe.getDownloader(); - channelId = urlIdHandler.getId(url); - String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId - + "?client_id=" + SoundcloudParsingHelper.clientId(); + channelId = getUrlIdHandler().getId(getOriginalUrl()); + String apiUrl = "https://api.soundcloud.com/users/" + channelId + + "?client_id=" + SoundcloudParsingHelper.clientId(); String response = dl.download(apiUrl); channel = new JSONObject(response); } + @Override + public String getCleanUrl() { + try { + return channel.getString("permalink_url"); + } catch (Exception e) { + return getOriginalUrl(); + } + } + @Override public String getChannelId() { return channelId; @@ -56,32 +65,6 @@ public String getBannerUrl() throws ParsingException { } } - @Override - public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Downloader dl = NewPipe.getDownloader(); - - String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=10" - + "&offset=0" - + "&linked_partitioning=1"; - - String response = dl.download(apiUrl); - JSONObject responseObject = new JSONObject(response); - - nextUrl = responseObject.getString("next_href") - + "&client_id=" + SoundcloudParsingHelper.clientId() - + "&linked_partitioning=1"; - - JSONArray responseCollection = responseObject.getJSONArray("collection"); - for (int i = 0; i < responseCollection.length(); i++) { - JSONObject track = responseCollection.getJSONObject(i); - collector.commit(new SoundcloudStreamInfoItemExtractor(track)); - } - return collector; - } - @Override public long getSubscriberCount() { return channel.getLong("followers_count"); @@ -93,26 +76,27 @@ public String getFeedUrl() throws ParsingException { } @Override - public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { - if (nextUrl.equals("")) { - throw new ExtractionException("Channel doesn't have more streams"); - } - - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Downloader dl = NewPipe.getDownloader(); + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); - String response = dl.download(nextUrl); - JSONObject responseObject = new JSONObject(response); - - nextUrl = responseObject.getString("next_href") - + "&client_id=" + SoundcloudParsingHelper.clientId() + String apiUrl = "https://api-v2.soundcloud.com/users/" + getChannelId() + "/tracks" + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=20" + "&linked_partitioning=1"; - JSONArray responseCollection = responseObject.getJSONArray("collection"); - for (int i = 0; i < responseCollection.length(); i++) { - JSONObject track = responseCollection.getJSONObject(i); - collector.commit(new SoundcloudStreamInfoItemExtractor(track)); - } + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); return collector; } + + @Override + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { + throw new ExtractionException("Channel doesn't have more streams"); + } + + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); + + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index 160de129aa..b676eb65f9 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -31,7 +31,7 @@ public long getSubscriberCount() { } @Override - public long getViewCount() { + public long getStreamCount() { return searchResult.getLong("track_count"); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java index be6c207545..4c96079c69 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -1,10 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.json.JSONObject; import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -21,47 +18,28 @@ public static SoundcloudChannelUrlIdHandler getInstance() { @Override public String getUrl(String channelId) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId - + "?client_id=" + SoundcloudParsingHelper.clientId()); - JSONObject responseObject = new JSONObject(response); - - return responseObject.getString("permalink_url"); + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + channelId); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public String getId(String siteUrl) throws ParsingException { + public String getId(String url) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(siteUrl); - Document doc = Jsoup.parse(response); - - Element androidElement = doc.select("meta[property=al:android:url]").first(); - String id = androidElement.attr("content").substring(19); - - return id; + return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public String cleanUrl(String siteUrl) throws ParsingException { + public String cleanUrl(String complexUrl) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(siteUrl); - Document doc = Jsoup.parse(response); + Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) + .select("meta[property=og:url]").first(); - Element ogElement = doc.select("meta[property=og:url]").first(); - String url = ogElement.attr("content"); - - return url; + return ogElement.attr("content"); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } @@ -71,6 +49,5 @@ public String cleanUrl(String siteUrl) throws ParsingException { public boolean acceptUrl(String channelUrl) { String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; return Parser.isMatch(regex, channelUrl.toLowerCase()); - } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 45af5c29e9..e78aad9618 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -1,13 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.concurrent.TimeUnit; - +import org.json.JSONArray; +import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -15,66 +9,135 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import java.io.IOException; +import java.net.URLEncoder; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + public class SoundcloudParsingHelper { + private static String clientId; + private SoundcloudParsingHelper() { } - public static final String clientId() throws ReCaptchaException, IOException, RegexException { + public static String clientId() throws ReCaptchaException, IOException, RegexException { + if (clientId != null && !clientId.isEmpty()) return clientId; + Downloader dl = NewPipe.getDownloader(); String response = dl.download("https://soundcloud.com"); Document doc = Jsoup.parse(response); + // TODO: Find a less heavy way to get the client_id + // Currently we are downloading a 1MB file (!) just to get the client_id, + // youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid. + // The embed mode has a way to get it, but we still have to download a heavy file (~800KB). Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first(); String js = dl.download(jsElement.attr("src")); - String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js); + clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js); return clientId; } - public static String toTimeAgoString(String time) throws ParsingException { + public static String toDateString(String time) throws ParsingException { try { - List times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30), - TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1), - TimeUnit.SECONDS.toMillis(1)); - List timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second"); + Date date; + // Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'. + try { + date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time); + } catch (Exception e) { + date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time); + } - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return newDateFormat.format(date); + } catch (ParseException e) { + throw new ParsingException(e.getMessage(), e); + } + } - long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime(); + /** + * Call the endpoint "/resolve" of the api.
+ * See https://developers.soundcloud.com/docs/api/reference#resolve + */ + public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { + String apiUrl = "https://api.soundcloud.com/resolve" + + "?url=" + URLEncoder.encode(url, "UTF-8") + + "&client_id=" + clientId(); - StringBuilder timeAgoString = new StringBuilder(); + return new JSONObject(NewPipe.getDownloader().download(apiUrl)); + } - for (int i = 0; i < times.size(); i++) { - Long current = times.get(i); - long currentAmount = timeAgo / current; - if (currentAmount > 0) { - timeAgoString.append(currentAmount).append(" ").append(timesString.get(i)) - .append(currentAmount != 1 ? "s ago" : " ago"); - break; - } - } - if (timeAgoString.toString().equals("")) { - timeAgoString.append("Just now"); - } - return timeAgoString.toString(); - } catch (ParseException e) { - throw new ParsingException(e.getMessage(), e); + /** + * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).
+ * + * @return the url resolved + */ + public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException { + + String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url=" + + URLEncoder.encode(apiUrl, "UTF-8")); + + return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href"); + } + + /** + * Fetch the embed player with the url and return the id (like the id from the json api).
+ * + * @return the id resolved + */ + public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException { + + String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url=" + + URLEncoder.encode(url, "UTF-8")); + return Parser.matchGroup1(",\"id\":(.*?),", response); + } + + /** + * Fetch the streams from the given api and commit each of them to the collector. + *

+ * This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always + * get MIN_ITEMS or more items. + * + * @param minItems the method will return only when it have extracted that many items (equal or more) + */ + public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); + + while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) { + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl); } + + return nextStreamsUrl; } - public static String toDateString(String time) throws ParsingException { + /** + * Fetch the streams from the given api and commit each of them to the collector. + * + * @return the next streams url, empty if don't have + */ + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + String response = NewPipe.getDownloader().download(apiUrl); + JSONObject responseObject = new JSONObject(response); + + JSONArray responseCollection = responseObject.getJSONArray("collection"); + for (int i = 0; i < responseCollection.length(); i++) { + collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); + } + + String nextStreamsUrl; try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - Date date = dateFormat.parse(time); - SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); - return newDateFormat.format(date); - } catch (ParseException e) { - throw new ParsingException(e.getMessage(), e); + nextStreamsUrl = responseObject.getString("next_href"); + if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); + } catch (Exception ignored) { + nextStreamsUrl = ""; } - } + return nextStreamsUrl; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 01a0bee4e9..08538a0d18 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -1,38 +1,46 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.util.List; - -import org.json.JSONArray; import org.json.JSONObject; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; +import java.io.IOException; + @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private String playlistId; private JSONObject playlist; - private List nextTracks; - public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException { - super(urlIdHandler, url, serviceId); + public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + @Override + public void fetchPage() throws IOException, ExtractionException { Downloader dl = NewPipe.getDownloader(); - playlistId = urlIdHandler.getId(url); - String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId - + "?client_id=" + SoundcloudParsingHelper.clientId(); + playlistId = getUrlIdHandler().getId(getOriginalUrl()); + String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&representation=compact"; String response = dl.download(apiUrl); playlist = new JSONObject(response); } + @Override + public String getCleanUrl() { + try { + return playlist.getString("permalink_url"); + } catch (Exception e) { + return getOriginalUrl(); + } + } + @Override public String getPlaylistId() { return playlistId; @@ -69,61 +77,33 @@ public String getUploaderAvatarUrl() { } @Override - public long getStreamsCount() { + public long getStreamCount() { return playlist.getLong("track_count"); } @Override - public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Downloader dl = NewPipe.getDownloader(); + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); - String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId - + "?client_id=" + SoundcloudParsingHelper.clientId(); + // Note the "api", NOT "api-v2" + String apiUrl = "https://api.soundcloud.com/playlists/" + getPlaylistId() + "/tracks" + + "?client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=20" + + "&linked_partitioning=1"; - String response = dl.download(apiUrl); - JSONObject responseObject = new JSONObject(response); - JSONArray responseCollection = responseObject.getJSONArray("collection"); - - for (int i = 0; i < responseCollection.length(); i++) { - JSONObject track = responseCollection.getJSONObject(i); - try { - collector.commit(new SoundcloudStreamInfoItemExtractor(track)); - } catch (Exception e) { - nextTracks.add(track.getString("id")); - } - } + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); return collector; } @Override - public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException { - if (nextTracks.equals(null)) { - return null; - } - - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Downloader dl = NewPipe.getDownloader(); - - // TODO: Do this per 10 tracks, instead of all tracks at once - String apiUrl = "https://api-v2.soundcloud.com/tracks?ids="; - for (String id : nextTracks) { - apiUrl += id; - if (!id.equals(nextTracks.get(nextTracks.size() - 1))) { - apiUrl += ","; - } + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { + throw new ExtractionException("Playlist doesn't have more streams"); } - apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); - String response = dl.download(apiUrl); - JSONObject responseObject = new JSONObject(response); - JSONArray responseCollection = responseObject.getJSONArray("collection"); + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); - for (int i = 0; i < responseCollection.length(); i++) { - JSONObject track = responseCollection.getJSONObject(i); - collector.commit(new SoundcloudStreamInfoItemExtractor(track)); - } - nextTracks = null; - return collector; + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java index 8244a8146f..4b5b7ee3e7 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -1,10 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.json.JSONObject; import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -21,13 +18,7 @@ public static SoundcloudPlaylistUrlIdHandler getInstance() { @Override public String getUrl(String listId) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId - + "?client_id=" + SoundcloudParsingHelper.clientId()); - JSONObject responseObject = new JSONObject(response); - - return responseObject.getString("permalink_url"); + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + listId); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } @@ -36,15 +27,7 @@ public String getUrl(String listId) throws ParsingException { @Override public String getId(String url) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(url); - Document doc = Jsoup.parse(response); - - Element androidElement = doc.select("meta[property=al:android:url]").first(); - String id = androidElement.attr("content").substring(23); - - return id; + return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } @@ -53,15 +36,10 @@ public String getId(String url) throws ParsingException { @Override public String cleanUrl(String complexUrl) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(complexUrl); - Document doc = Jsoup.parse(response); - - Element ogElement = doc.select("meta[property=og:url]").first(); - String url = ogElement.attr("content"); + Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) + .select("meta[property=og:url]").first(); - return url; + return ogElement.attr("content"); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java index 8837b933bc..757ff51b80 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java @@ -1,9 +1,5 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.net.URLEncoder; -import java.util.EnumSet; - import org.json.JSONArray; import org.json.JSONObject; import org.schabi.newpipe.extractor.Downloader; @@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.SearchEngine; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.EnumSet; + public class SoundcloudSearchEngine extends SearchEngine { public static final String CHARSET_UTF_8 = "UTF-8"; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 9a0ba43118..80c565ac86 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -13,61 +13,48 @@ public class SoundcloudService extends StreamingService { - public SoundcloudService(int id) { - super(id); + public SoundcloudService(int id, String name) { + super(id, name); } @Override - public ServiceInfo getServiceInfo() { - ServiceInfo serviceInfo = new ServiceInfo(); - serviceInfo.name = "Soundcloud"; - return serviceInfo; - } - - @Override - public StreamExtractor getStreamExtractorInstance(String url) - throws ExtractionException, IOException { - UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance(); - if (urlIdHandler.acceptUrl(url)) { - return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId()); - } else { - throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL"); - } - } - - @Override - public SearchEngine getSearchEngineInstance() { + public SearchEngine getSearchEngine() { return new SoundcloudSearchEngine(getServiceId()); } @Override - public UrlIdHandler getStreamUrlIdHandlerInstance() { + public UrlIdHandler getStreamUrlIdHandler() { return SoundcloudStreamUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getChannelUrlIdHandlerInstance() { + public UrlIdHandler getChannelUrlIdHandler() { return SoundcloudChannelUrlIdHandler.getInstance(); } - @Override - public UrlIdHandler getPlaylistUrlIdHandlerInstance() { + public UrlIdHandler getPlaylistUrlIdHandler() { return SoundcloudPlaylistUrlIdHandler.getInstance(); } + + @Override + public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException { + return new SoundcloudStreamExtractor(this, url); + } + @Override - public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException { - return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId()); + public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { + return new SoundcloudChannelExtractor(this, url, nextStreamsUrl); } @Override - public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException { - return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId()); + public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { + return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl); } @Override - public SuggestionExtractor getSuggestionExtractorInstance() { + public SuggestionExtractor getSuggestionExtractor() { return new SoundcloudSuggestionExtractor(getServiceId()); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 61eaa5c538..f75ec06db2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -1,19 +1,14 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.util.List; -import java.util.Vector; - import org.json.JSONArray; import org.json.JSONObject; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; @@ -21,33 +16,39 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class SoundcloudStreamExtractor extends StreamExtractor { - private String pageUrl; - private String trackId; private JSONObject track; - public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, pageUrl, serviceId); - - Downloader dl = NewPipe.getDownloader(); - - trackId = urlIdHandler.getId(pageUrl); - String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId - + "?client_id=" + SoundcloudParsingHelper.clientId(); + public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + super(service, url); + } - String response = dl.download(apiUrl); - track = new JSONObject(response); + @Override + public void fetchPage() throws IOException, ExtractionException { + track = SoundcloudParsingHelper.resolveFor(getOriginalUrl()); if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) { throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy")); } } + @Override + public String getCleanUrl() { + try { + return track.getString("permalink_url"); + } catch (Exception e) { + return getOriginalUrl(); + } + } + @Override public String getId() { - return trackId; + return track.getInt("id") + ""; } @Override @@ -96,11 +97,11 @@ public String getDashMpdUrl() { } @Override - public List getAudioStreams() throws ReCaptchaException, IOException, RegexException { - Vector audioStreams = new Vector<>(); + public List getAudioStreams() throws IOException, ExtractionException { + List audioStreams = new ArrayList<>(); Downloader dl = NewPipe.getDownloader(); - String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams" + String apiUrl = "https://api.soundcloud.com/i1/tracks/" + getId() + "/streams" + "?client_id=" + SoundcloudParsingHelper.clientId(); String response = dl.download(apiUrl); @@ -113,20 +114,20 @@ public List getAudioStreams() throws ReCaptchaException, IOExceptio } @Override - public List getVideoStreams() { + public List getVideoStreams() throws IOException, ExtractionException { return null; } @Override - public List getVideoOnlyStreams() { - return null; + public List getVideoOnlyStreams() throws IOException, ExtractionException { + return null; } @Override public int getTimeStamp() throws ParsingException { String timeStamp; try { - timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl()); } catch (Parser.RegexException e) { // catch this instantly since an url does not necessarily have to have a time stamp @@ -190,16 +191,16 @@ public int getDislikeCount() { } @Override - public StreamInfoItemExtractor getNextVideo() { + public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException { return null; } @Override - public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); Downloader dl = NewPipe.getDownloader(); - String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related" + String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related" + "?client_id=" + SoundcloudParsingHelper.clientId(); String response = dl.download(apiUrl); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index 001315c098..3b768808cd 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -35,7 +35,7 @@ public String getUploader() { @Override public String getUploadDate() throws ParsingException { - return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at")); + return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at")); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java index 2badb8ee51..0baf47c387 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -1,10 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.json.JSONObject; import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,6 +10,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); + private SoundcloudStreamUrlIdHandler() { } @@ -23,13 +21,7 @@ public static SoundcloudStreamUrlIdHandler getInstance() { @Override public String getUrl(String videoId) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId - + "?client_id=" + SoundcloudParsingHelper.clientId()); - JSONObject responseObject = new JSONObject(response); - - return responseObject.getString("permalink_url"); + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + videoId); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } @@ -38,15 +30,7 @@ public String getUrl(String videoId) throws ParsingException { @Override public String getId(String url) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(url); - Document doc = Jsoup.parse(response); - - Element androidElement = doc.select("meta[property=al:android:url]").first(); - String id = androidElement.attr("content").substring(20); - - return id; + return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } @@ -55,15 +39,10 @@ public String getId(String url) throws ParsingException { @Override public String cleanUrl(String complexUrl) throws ParsingException { try { - Downloader dl = NewPipe.getDownloader(); - - String response = dl.download(complexUrl); - Document doc = Jsoup.parse(response); - - Element ogElement = doc.select("meta[property=og:url]").first(); - String url = ogElement.attr("content"); + Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) + .select("meta[property=og:url]").first(); - return url; + return ogElement.attr("content"); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java index 6af0fdbddb..1dbee93cec 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; - import org.json.JSONArray; import org.json.JSONObject; import org.schabi.newpipe.extractor.Downloader; @@ -13,6 +8,11 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + public class SoundcloudSuggestionExtractor extends SuggestionExtractor { public static final String CHARSET_UTF_8 = "UTF-8"; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 1e1cc55527..8a68f2aaa6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -43,7 +43,7 @@ public class ItagItem { // Disable Opus codec as it's not well supported in older devices // new ItagItem(249, AUDIO, WEBMA, 50), // new ItagItem(250, AUDIO, WEBMA, 70), -// new ItagItem(251, AUDIO, WEBMA, 16), +// new ItagItem(251, AUDIO, WEBMA, 160), new ItagItem(171, AUDIO, WEBMA, 128), new ItagItem(172, AUDIO, WEBMA, 256), new ItagItem(139, AUDIO, M4A, 48), diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 56cad383d2..29f7117272 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -8,7 +8,7 @@ import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -44,6 +44,7 @@ @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; + private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private Document doc; /** @@ -51,29 +52,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor { */ private Document nextStreamsAjax; - /*////////////////////////////////////////////////////////////////////////// - // Variables for cache purposes (not "select" the current document all over again) - //////////////////////////////////////////////////////////////////////////*/ - private String channelId; - private String channelName; - private String avatarUrl; - private String bannerUrl; - private String feedUrl; - private long subscriberCount = -1; - - public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId); - fetchDocument(); + public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + + @Override + public void fetchPage() throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); + + String userUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS; + String pageContent = downloader.download(userUrl); + doc = Jsoup.parse(pageContent, userUrl); + + nextStreamsUrl = getNextStreamsUrlFrom(doc); + nextStreamsAjax = null; } @Override public String getChannelId() throws ParsingException { try { - if (channelId == null) { - channelId = getUrlIdHandler().getId(getUrl()); - } - - return channelId; + return getUrlIdHandler().getId(getCleanUrl()); } catch (Exception e) { throw new ParsingException("Could not get channel id"); } @@ -82,11 +80,7 @@ public String getChannelId() throws ParsingException { @Override public String getChannelName() throws ParsingException { try { - if (channelName == null) { - channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); - } - - return channelName; + return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); } catch (Exception e) { throw new ParsingException("Could not get channel name"); } @@ -95,11 +89,7 @@ public String getChannelName() throws ParsingException { @Override public String getAvatarUrl() throws ParsingException { try { - if (avatarUrl == null) { - avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src"); - } - - return avatarUrl; + return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src"); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -108,59 +98,47 @@ public String getAvatarUrl() throws ParsingException { @Override public String getBannerUrl() throws ParsingException { try { - if (bannerUrl == null) { - Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent); - - bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url; - } + Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); + String cssContent = el.html(); + String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent); - return bannerUrl; + return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url; } catch (Exception e) { throw new ParsingException("Could not get Banner", e); } } - @Override - public StreamInfoItemCollector getStreams() throws ParsingException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); - collectStreamsFrom(collector, ul); - return collector; - } @Override public long getSubscriberCount() throws ParsingException { - - if (subscriberCount == -1) { - Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); - if (el != null) { - subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text())); - } else { - throw new ParsingException("Could not get subscriber count"); - } + Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); + if (el != null) { + return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + } else { + throw new ParsingException("Could not get subscriber count"); } - - return subscriberCount; } @Override public String getFeedUrl() throws ParsingException { try { - if (feedUrl == null) { - String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id"); - feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId; - } - - return feedUrl; + String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id"); + return channelId == null ? "" : CHANNEL_FEED_BASE + channelId; } catch (Exception e) { throw new ParsingException("Could not get feed url", e); } } @Override - public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); + collectStreamsFrom(collector, ul); + return collector; + } + + @Override + public NextItemsResult getNextStreams() throws IOException, ExtractionException { if (!hasMoreStreams()) { throw new ExtractionException("Channel doesn't have more streams"); } @@ -169,7 +147,7 @@ public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOEx setupNextStreamsAjax(NewPipe.getDownloader()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); - return collector; + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); } private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { @@ -182,8 +160,7 @@ private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReC String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); if (!nextStreamsHtmlDataRaw.isEmpty()) { - Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl); - nextStreamsUrl = getNextStreamsUrl(nextStreamsData); + nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl)); } else { nextStreamsUrl = ""; } @@ -192,7 +169,7 @@ private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReC } } - private String getNextStreamsUrl(Document d) throws ParsingException { + private String getNextStreamsUrlFrom(Document d) throws ParsingException { try { Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); if (button != null) { @@ -206,17 +183,6 @@ private String getNextStreamsUrl(Document d) throws ParsingException { } } - private void fetchDocument() throws IOException, ReCaptchaException, ParsingException { - Downloader downloader = NewPipe.getDownloader(); - - String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000"; - String pageContent = downloader.download(userUrl); - doc = Jsoup.parse(pageContent, userUrl); - - nextStreamsUrl = getNextStreamsUrl(doc); - nextStreamsAjax = null; - } - private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { collector.getItemList().clear(); @@ -230,7 +196,8 @@ public StreamType getStreamType() throws ParsingException { @Override public boolean isAd() throws ParsingException { - return !li.select("span[class*=\"icon-not-available\"]").isEmpty(); + return !li.select("span[class*=\"icon-not-available\"]").isEmpty() || + !li.select("span[class*=\"yt-badge-ad\"]").isEmpty(); } @Override @@ -259,7 +226,7 @@ public String getTitle() throws ParsingException { public int getDuration() throws ParsingException { try { return YoutubeParsingHelper.parseDurationString( - li.select("span[class=\"video-time\"]").first().text()); + li.select("span[class*=\"video-time\"]").first().text()); } catch (Exception e) { if (isLiveStream(li)) { // -1 for no duration diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java index 3caff88b07..7151a33b3c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java @@ -68,7 +68,7 @@ public long getSubscriberCount() throws ParsingException { } @Override - public long getViewCount() throws ParsingException { + public long getStreamCount() throws ParsingException { Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first(); if (metaEl == null) { return 0; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index 5e8813992b..36fe614c74 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -7,6 +7,7 @@ import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -23,39 +24,31 @@ @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - private Document doc = null; + private Document doc; /** * It's lazily initialized (when getNextStreams is called) */ - private Document nextStreamsAjax = null; + private Document nextStreamsAjax; - /*////////////////////////////////////////////////////////////////////////// - // Variables for cache purposes (not "select" the current document all over again) - //////////////////////////////////////////////////////////////////////////*/ - private String playlistId; - private String playlistName; - private String avatarUrl; - private String bannerUrl; + public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } - private long streamsCount; + @Override + public void fetchPage() throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); - private String uploaderUrl; - private String uploaderName; - private String uploaderAvatarUrl; + String pageContent = downloader.download(getCleanUrl()); + doc = Jsoup.parse(pageContent, getCleanUrl()); - public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException { - super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId); - fetchDocument(); + nextStreamsUrl = getNextStreamsUrlFrom(doc); + nextStreamsAjax = null; } @Override public String getPlaylistId() throws ParsingException { try { - if (playlistId == null) { - playlistId = getUrlIdHandler().getId(getUrl()); - } - - return playlistId; + return getUrlIdHandler().getId(getCleanUrl()); } catch (Exception e) { throw new ParsingException("Could not get playlist id"); } @@ -64,11 +57,7 @@ public String getPlaylistId() throws ParsingException { @Override public String getPlaylistName() throws ParsingException { try { - if (playlistName == null) { - playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text(); - } - - return playlistName; + return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text(); } catch (Exception e) { throw new ParsingException("Could not get playlist name"); } @@ -77,11 +66,7 @@ public String getPlaylistName() throws ParsingException { @Override public String getAvatarUrl() throws ParsingException { try { - if (avatarUrl == null) { - avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src"); - } - - return avatarUrl; + return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src"); } catch (Exception e) { throw new ParsingException("Could not get playlist avatar"); } @@ -90,18 +75,16 @@ public String getAvatarUrl() throws ParsingException { @Override public String getBannerUrl() throws ParsingException { try { - if (bannerUrl == null) { - Element el = doc.select("div[id=\"gh-banner\"] style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent); - if (url.contains("s.ytimg.com")) { - bannerUrl = null; - } else { - bannerUrl = url.substring(0, url.indexOf(");")); - } + Element el = doc.select("div[id=\"gh-banner\"] style").first(); + String cssContent = el.html(); + String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent); + if (url.contains("s.ytimg.com")) { + return null; + } else { + return url.substring(0, url.indexOf(");")); } - return bannerUrl; + } catch (Exception e) { throw new ParsingException("Could not get playlist Banner"); } @@ -110,11 +93,7 @@ public String getBannerUrl() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { try { - if (uploaderUrl == null) { - uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href"); - } - - return uploaderUrl; + return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name"); } @@ -123,11 +102,7 @@ public String getUploaderUrl() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - if (uploaderName == null) { - uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); - } - - return uploaderName; + return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name"); } @@ -136,54 +111,46 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - if (uploaderAvatarUrl == null) { - uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src"); - } - - return uploaderAvatarUrl; + return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar"); } } @Override - public long getStreamsCount() throws ParsingException { - if (streamsCount <= 0) { - String input; - - try { - input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text(); - } catch (IndexOutOfBoundsException e) { - throw new ParsingException("Could not get video count from playlist", e); - } + public long getStreamCount() throws ParsingException { + String input; - try { - streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input)); - } catch (NumberFormatException e) { - // When there's no videos in a playlist, there's no number in the "innerHtml", - // all characters that is not a number is removed, so we try to parse a empty string - if (!input.isEmpty()) { - streamsCount = 0; - } else { - throw new ParsingException("Could not handle input: " + input, e); - } - } + try { + input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text(); + } catch (IndexOutOfBoundsException e) { + throw new ParsingException("Could not get video count from playlist", e); } - return streamsCount; + try { + return Long.parseLong(Utils.removeNonDigitCharacters(input)); + } catch (NumberFormatException e) { + // When there's no videos in a playlist, there's no number in the "innerHtml", + // all characters that is not a number is removed, so we try to parse a empty string + if (!input.isEmpty()) { + return 0; + } else { + throw new ParsingException("Could not handle input: " + input, e); + } + } } @Override - public StreamInfoItemCollector getStreams() throws ParsingException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); collectStreamsFrom(collector, tbody); return collector; } @Override - public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException { - if (!hasMoreStreams()){ + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { throw new ExtractionException("Playlist doesn't have more streams"); } @@ -191,7 +158,7 @@ public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOEx setupNextStreamsAjax(NewPipe.getDownloader()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); - return collector; + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); } private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { @@ -204,8 +171,7 @@ private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReC String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); if (!nextStreamsHtmlDataRaw.isEmpty()) { - final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl); - nextStreamsUrl = getNextStreamsUrl(nextStreamsData); + nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl)); } else { nextStreamsUrl = ""; } @@ -214,17 +180,7 @@ private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReC } } - private void fetchDocument() throws IOException, ReCaptchaException, ParsingException { - Downloader downloader = NewPipe.getDownloader(); - - String pageContent = downloader.download(getUrl()); - doc = Jsoup.parse(pageContent, getUrl()); - - nextStreamsUrl = getNextStreamsUrl(doc); - nextStreamsAjax = null; - } - - private String getNextStreamsUrl(Document d) throws ParsingException { + private String getNextStreamsUrlFrom(Document d) throws ParsingException { try { Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); if (button != null) { @@ -241,7 +197,7 @@ private String getNextStreamsUrl(Document d) throws ParsingException { private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { collector.getItemList().clear(); - final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance(); + final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler(); for (final Element li : element.children()) { collector.commit(new StreamInfoItemExtractor() { @Override @@ -252,7 +208,7 @@ public StreamType getStreamType() throws ParsingException { @Override public String getWebPageUrl() throws ParsingException { try { - return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id")); + return streamUrlIdHandler.getUrl(li.attr("data-video-id")); } catch (Exception e) { throw new ParsingException("Could not get web page url for the video", e); } @@ -300,7 +256,7 @@ public long getViewCount() throws ParsingException { @Override public String getThumbnailUrl() throws ParsingException { try { - return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg"; + return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg"; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 082a1c6da1..eb89260247 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -34,61 +34,48 @@ public class YoutubeService extends StreamingService { - public YoutubeService(int id) { - super(id); + public YoutubeService(int id, String name) { + super(id, name); } @Override - public ServiceInfo getServiceInfo() { - ServiceInfo serviceInfo = new ServiceInfo(); - serviceInfo.name = "Youtube"; - return serviceInfo; - } - - @Override - public StreamExtractor getStreamExtractorInstance(String url) - throws ExtractionException, IOException { - UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance(); - if (urlIdHandler.acceptUrl(url)) { - return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId()); - } else { - throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); - } - } - - @Override - public SearchEngine getSearchEngineInstance() { + public SearchEngine getSearchEngine() { return new YoutubeSearchEngine(getServiceId()); } @Override - public UrlIdHandler getStreamUrlIdHandlerInstance() { + public UrlIdHandler getStreamUrlIdHandler() { return YoutubeStreamUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getChannelUrlIdHandlerInstance() { + public UrlIdHandler getChannelUrlIdHandler() { return YoutubeChannelUrlIdHandler.getInstance(); } - @Override - public UrlIdHandler getPlaylistUrlIdHandlerInstance() { + public UrlIdHandler getPlaylistUrlIdHandler() { return YoutubePlaylistUrlIdHandler.getInstance(); } + + @Override + public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException { + return new YoutubeStreamExtractor(this, url); + } + @Override - public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException { - return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId()); + public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { + return new YoutubeChannelExtractor(this, url, nextStreamsUrl); } @Override - public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException { - return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId()); + public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { + return new YoutubePlaylistExtractor(this, url, nextStreamsUrl); } @Override - public SuggestionExtractor getSuggestionExtractorInstance() { + public SuggestionExtractor getSuggestionExtractor() { return new YoutubeSuggestionExtractor(getServiceId()); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 197e9bfe12..7fe23053ba 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -10,7 +10,7 @@ import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -26,9 +26,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Vector; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -80,12 +80,9 @@ public class LiveStreamException extends ContentNotAvailableException { /*//////////////////////////////////////////////////////////////////////////*/ private Document doc; - private final String dirtyUrl; - public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId); - dirtyUrl = pageUrl; - fetchDocument(); + public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + super(service, url); } /*////////////////////////////////////////////////////////////////////////// @@ -95,7 +92,7 @@ public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int ser @Override public String getId() throws ParsingException { try { - return getUrlIdHandler().getId(getUrl()); + return getUrlIdHandler().getId(getCleanUrl()); } catch (Exception e) { throw new ParsingException("Could not get stream id"); } @@ -238,8 +235,8 @@ public String getDashMpdUrl() throws ParsingException { } @Override - public List getAudioStreams() throws ParsingException { - Vector audioStreams = new Vector<>(); + public List getAudioStreams() throws IOException, ExtractionException { + List audioStreams = new ArrayList<>(); try { String encodedUrlMap; // playerArgs could be null if the video is age restricted @@ -288,8 +285,8 @@ public List getAudioStreams() throws ParsingException { } @Override - public List getVideoStreams() throws ParsingException { - Vector videoStreams = new Vector<>(); + public List getVideoStreams() throws IOException, ExtractionException { + List videoStreams = new ArrayList<>(); try { String encodedUrlMap; @@ -342,8 +339,8 @@ public List getVideoStreams() throws ParsingException { } @Override - public List getVideoOnlyStreams() throws ParsingException { - Vector videoOnlyStreams = new Vector<>(); + public List getVideoOnlyStreams() throws IOException, ExtractionException { + List videoOnlyStreams = new ArrayList<>(); try { String encodedUrlMap; @@ -405,7 +402,7 @@ public List getVideoOnlyStreams() throws ParsingException { public int getTimeStamp() throws ParsingException { String timeStamp; try { - timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl); + timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl()); } catch (Parser.RegexException e) { // catch this instantly since an url does not necessarily have to have a time stamp @@ -516,7 +513,7 @@ public int getDislikeCount() throws ParsingException { } @Override - public StreamInfoItemExtractor getNextVideo() throws ParsingException { + public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException { try { return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() .select("li").first()); @@ -526,9 +523,9 @@ public StreamInfoItemExtractor getNextVideo() throws ParsingException { } @Override - public StreamInfoItemCollector getRelatedVideos() throws ParsingException { + public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException { try { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); Element ul = doc.select("ul[id=\"watch-related\"]").first(); if (ul != null) { for (Element li : ul.children()) { @@ -617,11 +614,12 @@ public String getErrorMessage() { // cached values private static volatile String decryptionCode = ""; - private void fetchDocument() throws IOException, ReCaptchaException, ParsingException { + @Override + public void fetchPage() throws IOException, ExtractionException { Downloader downloader = NewPipe.getDownloader(); - String pageContent = downloader.download(getUrl()); - doc = Jsoup.parse(pageContent, getUrl()); + String pageContent = downloader.download(getCleanUrl()); + doc = Jsoup.parse(pageContent, getCleanUrl()); JSONObject ytPlayerConfig; String playerUrl; @@ -632,7 +630,7 @@ private void fetchDocument() throws IOException, ReCaptchaException, ParsingExce // Check if the video is age restricted if (pageContent.contains(" suggestionList( - String query, String contentCountry) - throws ExtractionException, IOException { - List suggestions = new ArrayList<>(); - + public List suggestionList(String query, String contentCountry) throws IOException, ExtractionException { Downloader dl = NewPipe.getDownloader(); + List suggestions = new ArrayList<>(); String url = "https://suggestqueries.google.com/complete/search" - + "?client=" + "" - + "&output=" + "toolbar" + + "?client=" + "firefox" // 'toolbar' for xml + "&ds=" + "yt" + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8) + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); - String response = dl.download(url); - - //TODO: Parse xml data using Jsoup not done - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder; - org.w3c.dom.Document doc = null; - - try { - dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource( - new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8)))); - doc.getDocumentElement().normalize(); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new ParsingException("Could not parse document."); - } - try { - NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); - for (int temp = 0; temp < nList.getLength(); temp++) { - - NodeList nList1 = doc.getElementsByTagName("suggestion"); - Node nNode1 = nList1.item(temp); - if (nNode1.getNodeType() == Node.ELEMENT_NODE) { - org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; - suggestions.add(eElement.getAttribute("data")); - } - } - return suggestions; + JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1); + for (Object suggestion : suggestionsArray) suggestions.add(suggestion.toString()); } catch (Exception e) { - throw new ParsingException("Could not get suggestions form document.", e); + throw new ParsingException("Could not parse suggestions response.", e); } + + return suggestions; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index c7606cee07..d14e565195 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -13,7 +13,7 @@ public Stream(String url, int format) { } /** - * Reveals whether two streams are the same, but have different urls + * Reveals whether two streams have the same stats (format and bitrate, for example) */ public boolean equalStats(Stream cmp) { return cmp != null && format == cmp.format; diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 1cf4c1fe5b..c70ad36d7c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -21,9 +21,10 @@ */ import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import java.io.IOException; import java.util.List; @@ -33,8 +34,14 @@ */ public abstract class StreamExtractor extends Extractor { - public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) { - super(urlIdHandler, serviceId, url); + public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { + super(service, url); + fetchPage(); + } + + @Override + protected UrlIdHandler getUrlIdHandler() throws ParsingException { + return getService().getStreamUrlIdHandler(); } public abstract String getId() throws ParsingException; @@ -48,22 +55,22 @@ public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) { public abstract String getUploadDate() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException; public abstract String getUploaderThumbnailUrl() throws ParsingException; - public abstract List getAudioStreams() throws ParsingException, ReCaptchaException, IOException; - public abstract List getVideoStreams() throws ParsingException; - public abstract List getVideoOnlyStreams() throws ParsingException; + public abstract List getAudioStreams() throws IOException, ExtractionException; + public abstract List getVideoStreams() throws IOException, ExtractionException; + public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; public abstract String getDashMpdUrl() throws ParsingException; public abstract int getAgeLimit() throws ParsingException; public abstract String getAverageRating() throws ParsingException; public abstract int getLikeCount() throws ParsingException; public abstract int getDislikeCount() throws ParsingException; - public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException; - public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException; + public abstract StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException; + public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException; public abstract StreamType getStreamType() throws ParsingException; /** * Analyses the webpage's document and extracts any error message there might be. * - * @return Error message; null if there is no error message. + * @return Error message; null if there is no error message. */ public abstract String getErrorMessage(); } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 6168ae26e5..2e0242ea16 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -2,14 +2,18 @@ import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; -import java.util.Vector; /* * Created by Christian Schabesberger on 26.08.15. @@ -46,11 +50,23 @@ public static class StreamExtractException extends ExtractionException { public StreamInfo() { } + public static StreamInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static StreamInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url); + } + + public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + return getInfo(service.getStreamExtractor(url)); + } + /** * Fills out the video info fields which are common to all services. * Probably needs to be overridden by subclasses */ - public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException { + public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException { StreamInfo streamInfo = new StreamInfo(); try { @@ -80,7 +96,7 @@ private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtr // if one of these is not available an exception is meant to be thrown directly into the frontend. streamInfo.service_id = extractor.getServiceId(); - streamInfo.url = extractor.getUrl(); + streamInfo.url = extractor.getCleanUrl(); streamInfo.stream_type = extractor.getStreamType(); streamInfo.id = extractor.getId(); streamInfo.name = extractor.getTitle(); @@ -128,15 +144,12 @@ private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor } // Lists can be null if a exception was thrown during extraction - if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>(); - if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>(); - if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>(); + if (streamInfo.video_streams == null) streamInfo.video_streams = new ArrayList<>(); + if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>(); + if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>(); if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) { try { - // Will try to find in the dash manifest for any stream that the ItagItem has (by the id), - // it has video, video only and audio streams and will only add to the list if it don't - // find a similar stream in the respective lists (calling Stream#equalStats). DashMpdParser.getStreams(streamInfo); } catch (Exception e) { // Sometimes we receive 403 (forbidden) error when trying to download the manifest, @@ -246,6 +259,8 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra streamInfo.addException(e); } + if (streamInfo.related_streams == null) streamInfo.related_streams = new ArrayList<>(); + return streamInfo; } @@ -278,7 +293,7 @@ public void addException(Exception e) { public int dislike_count = -1; public String average_rating; public StreamInfoItem next_video; - public List related_streams = new Vector<>(); + public List related_streams; //in seconds. some metadata is not passed using a StreamInfo object! public int start_position = 0; } diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 2fa6b249a6..27b6251686 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -53,7 +53,13 @@ public static class DashMpdParsingException extends ParsingException { } /** - * Download manifest and return nodelist with elements of tag "AdaptationSet" + * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest, + * then it will search for any stream that the ItagItem has (by the id). + *

+ * It has video, video only and audio streams and will only add to the list if it don't + * find a similar stream in the respective lists (calling {@link Stream#equalStats}). + * + * @param streamInfo where the parsed streams will be added */ public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { String dashDoc; diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index ff32c88b68..d81b7f8d94 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -9,7 +9,7 @@ private Utils() { * Remove all non-digit characters from a string.

* Examples:
*

+ *
  • $31,133.124 -> 31133124
    • * * @param toRemove string to remove non-digit chars * @return a string that contains only digits @@ -21,15 +21,23 @@ public static String removeNonDigitCharacters(String toRemove) { /** * Check if throwable have the cause */ - public static boolean hasCauseThrowable(Throwable throwable, Class causeToCheck) { + public static boolean hasCauseThrowable(Throwable throwable, Class... causesToCheck) { // Check if getCause is not the same as cause (the getCause is already the root), // as it will cause a infinite loop if it is - Throwable cause, getCause = throwable; + Throwable cause, getCause = throwable; + + for (Class causesEl : causesToCheck) { + if (throwable.getClass().isAssignableFrom(causesEl)) { + return true; + } + } while ((cause = throwable.getCause()) != null && getCause != cause) { getCause = cause; - if (cause.getClass().isAssignableFrom(causeToCheck)) { - return true; + for (Class causesEl : causesToCheck) { + if (cause.getClass().isAssignableFrom(causesEl)) { + return true; + } } } return false; diff --git a/src/test/java/org/schabi/newpipe/Downloader.java b/src/test/java/org/schabi/newpipe/Downloader.java index 870c17005d..e3e08cc5e5 100644 --- a/src/test/java/org/schabi/newpipe/Downloader.java +++ b/src/test/java/org/schabi/newpipe/Downloader.java @@ -14,7 +14,7 @@ import javax.net.ssl.HttpsURLConnection; -/** +/* * Created by Christian Schabesberger on 28.01.16. * * Copyright (C) Christian Schabesberger 2016 @@ -35,16 +35,17 @@ */ public class Downloader implements org.schabi.newpipe.extractor.Downloader { - + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; private static String mCookies = ""; private static Downloader instance = null; - private Downloader() {} + private Downloader() { + } public static Downloader getInstance() { - if(instance == null) { + if (instance == null) { synchronized (Downloader.class) { if (instance == null) { instance = new Downloader(); @@ -62,11 +63,14 @@ public static synchronized String getCookies() { return Downloader.mCookies; } - /**Download the text file at the supplied URL as in download(String), + /** + * Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. - * @param siteUrl the URL of the text file to return the contents of + * + * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language - * @return the contents of the specified text file*/ + * @return the contents of the specified text file + */ public String download(String siteUrl, String language) throws IOException, ReCaptchaException { Map requestProperties = new HashMap<>(); requestProperties.put("Accept-Language", language); @@ -74,29 +78,35 @@ public String download(String siteUrl, String language) throws IOException, ReCa } - /**Download the text file at the supplied URL as in download(String), + /** + * Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. - * @param siteUrl the URL of the text file to return the contents of + * + * @param siteUrl the URL of the text file to return the contents of * @param customProperties set request header properties * @return the contents of the specified text file - * @throws IOException*/ + * @throws IOException + */ public String download(String siteUrl, Map customProperties) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); Iterator it = customProperties.entrySet().iterator(); - while(it.hasNext()) { - Map.Entry pair = (Map.Entry)it.next(); - con.setRequestProperty((String)pair.getKey(), (String)pair.getValue()); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + con.setRequestProperty((String) pair.getKey(), (String) pair.getValue()); } return dl(con); } - /**Common functionality between download(String url) and download(String url, String language)*/ + /** + * Common functionality between download(String url) and download(String url, String language) + */ private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException { StringBuilder response = new StringBuilder(); BufferedReader in = null; try { + con.setReadTimeout(30 * 1000);// 30s con.setRequestMethod("GET"); con.setRequestProperty("User-Agent", USER_AGENT); @@ -108,13 +118,13 @@ private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaEx new InputStreamReader(con.getInputStream())); String inputLine; - while((inputLine = in.readLine()) != null) { + while ((inputLine = in.readLine()) != null) { response.append(inputLine); } - } catch(UnknownHostException uhe) {//thrown when there's no internet connection + } catch (UnknownHostException uhe) {//thrown when there's no internet connection throw new IOException("unknown host or no network", uhe); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); - } catch(Exception e) { + } catch (Exception e) { /* * HTTP 429 == Too Many Request * Receive from Youtube.com = ReCaptcha challenge request @@ -123,9 +133,10 @@ private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaEx if (con.getResponseCode() == 429) { throw new ReCaptchaException("reCaptcha Challenge requested"); } - throw new IOException(e); + + throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e); } finally { - if(in != null) { + if (in != null) { in.close(); } } @@ -133,10 +144,13 @@ private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaEx return response.toString(); } - /**Download (via HTTP) the text file located at the supplied URL, and return its contents. + /** + * Download (via HTTP) the text file located at the supplied URL, and return its contents. * Primarily intended for downloading web pages. + * * @param siteUrl the URL of the text file to download - * @return the contents of the specified text file*/ + * @return the contents of the specified text file + */ public String download(String siteUrl) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); diff --git a/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java b/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java new file mode 100644 index 0000000000..1c59e76f0d --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java @@ -0,0 +1,74 @@ +package org.schabi.newpipe.extractor; + +import org.junit.Test; + +import java.util.HashSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; +import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; + +public class NewPipeTest { + @Test + public void getAllServicesTest() throws Exception { + assertEquals(NewPipe.getServices().length, ServiceList.values().length); + } + + @Test + public void testAllServicesHaveDifferentId() throws Exception { + HashSet servicesId = new HashSet<>(); + for (StreamingService streamingService : NewPipe.getServices()) { + String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().name + ")"; + + assertTrue(errorMsg, servicesId.add(streamingService.getServiceId())); + } + } + + @Test + public void getServiceWithId() throws Exception { + assertEquals(NewPipe.getService(Youtube.getId()), Youtube.getService()); + assertEquals(NewPipe.getService(SoundCloud.getId()), SoundCloud.getService()); + + assertNotEquals(NewPipe.getService(SoundCloud.getId()), Youtube.getService()); + } + + @Test + public void getServiceWithName() throws Exception { + assertEquals(NewPipe.getService(Youtube.getServiceInfo().name), Youtube.getService()); + assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().name), SoundCloud.getService()); + + assertNotEquals(NewPipe.getService(Youtube.getServiceInfo().name), SoundCloud.getService()); + } + + @Test + public void getServiceWithUrl() throws Exception { + assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), Youtube.getService()); + assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), Youtube.getService()); + assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), Youtube.getService()); + assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud.getService()); + assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud.getService()); + assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud.getService()); + + assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), Youtube.getService()); + assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud.getService()); + } + + @Test + public void getIdWithServiceName() throws Exception { + assertEquals(NewPipe.getIdOfService(Youtube.getServiceInfo().name), Youtube.getId()); + assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), SoundCloud.getId()); + + assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), Youtube.getId()); + } + + @Test + public void getServiceNameWithId() throws Exception { + assertEquals(NewPipe.getNameOfService(Youtube.getId()), Youtube.getServiceInfo().name); + assertEquals(NewPipe.getNameOfService(SoundCloud.getId()), SoundCloud.getServiceInfo().name); + + assertNotEquals(NewPipe.getNameOfService(Youtube.getId()), SoundCloud.getServiceInfo().name); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 0338bb7689..3c98a5a2a2 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,16 +1,17 @@ package org.schabi.newpipe.extractor.services.youtube; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertTrue; - import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; -/** +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/* * Created by Christian Schabesberger on 12.09.16. * * Copyright (C) Christian Schabesberger 2015 @@ -41,8 +42,8 @@ public class YoutubeChannelExtractorTest { @Before public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); - extractor = NewPipe.getService("Youtube") - .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); + extractor = Youtube.getService() + .getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); } @Test @@ -61,7 +62,7 @@ public void testGetAvatarUrl() throws Exception { } @Test - public void testGetBannerurl() throws Exception { + public void testGetBannerUrl() throws Exception { assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3")); } @@ -81,9 +82,10 @@ public void testGetStreamsErrors() throws Exception { } @Test - public void testHasNextPage() throws Exception { - // this particular example (link) has a next page !!! - assertTrue("no next page link found", extractor.hasMoreStreams()); + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("don't have more streams", extractor.hasMoreStreams()); } @Test @@ -92,16 +94,11 @@ public void testGetSubscriberCount() throws Exception { } @Test - public void testGetNextPage() throws Exception { - extractor = NewPipe.getService("Youtube") - .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); - assertTrue("next page didn't have content", !extractor.getStreams().getItemList().isEmpty()); + public void testGetNextStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } - @Test - public void testGetNextNextPageUrl() throws Exception { - extractor = NewPipe.getService("Youtube") - .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); - assertTrue("next page didn't have content", extractor.hasMoreStreams()); - } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java new file mode 100644 index 0000000000..45fc46d1b4 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -0,0 +1,98 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/** + * Test for {@link PlaylistExtractor} + */ + +public class YoutubePlaylistExtractorTest { + private PlaylistExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = Youtube.getService() + .getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y"); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetId() throws Exception { + assertEquals(extractor.getPlaylistId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y"); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getPlaylistName(), "important videos"); + } + + @Test + public void testGetAvatarUrl() throws Exception { + assertTrue(extractor.getAvatarUrl(), extractor.getAvatarUrl().contains("yt")); + } + + @Test + public void testGetBannerUrl() throws Exception { + assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt")); + } + + @Test + public void testGetUploaderUrl() throws Exception { + assertTrue(extractor.getUploaderUrl(), extractor.getUploaderUrl().contains("youtube.com")); + } + + @Test + public void testGetUploaderName() throws Exception { + assertTrue(extractor.getUploaderName(), !extractor.getUploaderName().isEmpty()); + } + + @Test + public void testGetUploaderAvatarUrl() throws Exception { + assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("yt")); + } + + @Test + public void testGetStreamsCount() throws Exception { + assertTrue("error in the streams count", extractor.getStreamCount() > 100); + } + + @Test + public void testGetStreams() throws Exception { + assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("extractor didn't have more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); + } + +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index ca01d2295c..e18c0830c7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -9,11 +9,12 @@ import java.util.EnumSet; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; -/** +/* * Created by Christian Schabesberger on 29.12.15. * * Copyright (C) Christian Schabesberger 2015 @@ -42,7 +43,7 @@ public class YoutubeSearchEngineAllTest { @Before public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); - SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); + SearchEngine engine = Youtube.getService().getSearchEngine(); // Youtube will suggest "asdf" instead of "asdgff" // keep in mind that the suggestions can change by country (the parameter "de") diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java index df0d0bb033..868b2b83c0 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java @@ -10,12 +10,13 @@ import java.util.EnumSet; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; -/** +/* * Created by Christian Schabesberger on 29.12.15. * * Copyright (C) Christian Schabesberger 2015 @@ -44,7 +45,7 @@ public class YoutubeSearchEngineChannelTest { @Before public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); - SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); + SearchEngine engine = Youtube.getService().getSearchEngine(); // Youtube will suggest "gronkh" instead of "grrunkh" // keep in mind that the suggestions can change by country (the parameter "de") @@ -59,7 +60,9 @@ public void testResultList() { @Test public void testChannelItemType() { - assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.CHANNEL); + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.CHANNEL, infoItem.info_type); + } } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java index 74334a3e56..da67ea7663 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java @@ -2,6 +2,7 @@ import org.junit.Before; import org.junit.Test; + import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; @@ -10,12 +11,13 @@ import java.util.EnumSet; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; -/** +/* * Created by Christian Schabesberger on 29.12.15. * * Copyright (C) Christian Schabesberger 2015 @@ -44,7 +46,7 @@ public class YoutubeSearchEngineStreamTest { @Before public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); - SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance(); + SearchEngine engine = Youtube.getService().getSearchEngine(); // Youtube will suggest "results" instead of "rsults", // keep in mind that the suggestions can change by country (the parameter "de") @@ -58,8 +60,10 @@ public void testResultList() { } @Test - public void testChannelItemType() { - assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.STREAM); + public void testStreamItemType() { + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.STREAM, infoItem.info_type); + } } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 8e2c7be780..8c3976e3d6 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -1,21 +1,23 @@ package org.schabi.newpipe.extractor.services.youtube; -import static junit.framework.Assert.assertTrue; - -import java.io.IOException; - import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; -/** +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/* * Created by Christian Schabesberger on 30.12.15. * * Copyright (C) Christian Schabesberger 2015 @@ -45,8 +47,7 @@ public class YoutubeStreamExtractorDefaultTest { @Before public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); - extractor = NewPipe.getService("Youtube") - .getStreamExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A"); + extractor = Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A"); } @Test @@ -56,10 +57,8 @@ public void testGetInvalidTimeStamp() throws ParsingException { } @Test - public void testGetValidTimeStamp() throws ExtractionException, IOException { - StreamExtractor extractor = - NewPipe.getService("Youtube") - .getStreamExtractorInstance("https://youtu.be/FmG385_uUys?t=174"); + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor = Youtube.getService().getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); assertTrue(Integer.toString(extractor.getTimeStamp()), extractor.getTimeStamp() == 174); } @@ -113,12 +112,12 @@ public void testGetUploaderThumbnailUrl() throws ParsingException { } @Test - public void testGetAudioStreams() throws ParsingException, ReCaptchaException, IOException { + public void testGetAudioStreams() throws IOException, ExtractionException { assertTrue(!extractor.getAudioStreams().isEmpty()); } @Test - public void testGetVideoStreams() throws ParsingException { + public void testGetVideoStreams() throws IOException, ExtractionException { for(VideoStream s : extractor.getVideoStreams()) { assertTrue(s.url, s.url.contains(HTTPS)); @@ -138,4 +137,11 @@ public void testGetDashMpd() throws ParsingException { assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty()); } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos(); + assertFalse(relatedVideos.getItemList().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorGemaTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorGemaTest.java new file mode 100644 index 0000000000..ba164e8005 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorGemaTest.java @@ -0,0 +1,53 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.io.IOException; + +import static org.junit.Assert.fail; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/* + * Created by Christian Schabesberger on 30.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeVideoExtractorGema.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * This exception is only thrown in Germany. + * + * WARNING: Deactivate this Test Case before uploading it to Github, otherwise CI will fail. + */ +@Ignore +public class YoutubeStreamExtractorGemaTest { + + @Test + public void testGemaError() throws IOException, ExtractionException { + try { + NewPipe.init(Downloader.getInstance()); + Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8"); + + fail("GemaException should be thrown"); + } catch (YoutubeStreamExtractor.GemaException ignored) { + // Exception was thrown, Gema error detection is working. + } + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java new file mode 100644 index 0000000000..862eaa4864 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -0,0 +1,108 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/** + * Test for {@link YoutubeStreamUrlIdHandler} + */ +public class YoutubeStreamExtractorRestrictedTest { + public static final String HTTPS = "https://"; + private StreamExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = Youtube.getService() + .getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0"); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor= Youtube.getService() + .getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() == 174); + } + + @Test + public void testGetAgeLimit() throws ParsingException { + assertTrue(extractor.getAgeLimit() == 18); + } + + @Test + public void testGetTitle() throws ParsingException { + assertTrue(!extractor.getTitle().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertTrue(extractor.getDescription() != null); + } + + @Test + public void testGetUploader() throws ParsingException { + assertTrue(!extractor.getUploader().isEmpty()); + } + + @Test + public void testGetLength() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + @Test + public void testGetViews() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertTrue(extractor.getThumbnailUrl(), + extractor.getThumbnailUrl().contains(HTTPS)); + } + + @Test + public void testGetUploaderThumbnailUrl() throws ParsingException { + assertTrue(extractor.getUploaderThumbnailUrl(), + extractor.getUploaderThumbnailUrl().contains(HTTPS)); + } + + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + // audiostream not always necessary + assertTrue(!extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + for(VideoStream s : extractor.getVideoStreams()) { + assertTrue(s.url, + s.url.contains(HTTPS)); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.format), + 0 <= s.format && s.format <= 4); + } + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java new file mode 100644 index 0000000000..c0c77714c4 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java @@ -0,0 +1,117 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link YoutubeStreamUrlIdHandler} + */ +public class YoutubeStreamUrlIdHandlerTest { + private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew"; + private YoutubeStreamUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = YoutubeStreamUrlIdHandler.getInstance(); + NewPipe.init(Downloader.getInstance()); + } + + @Test(expected = NullPointerException.class) + public void getIdWithNullAsUrl() throws ParsingException { + urlIdHandler.getId(null); + } + + @Test(expected = FoundAdException.class) + public void getIdForAd() throws ParsingException { + urlIdHandler.getId(AD_URL); + } + + @Test + public void getIdForInvalidUrls() throws ParsingException { + List invalidUrls = new ArrayList<>(50); + invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d"); + invalidUrls.add("https://www.youtube.com/watchjZViOEv90d"); + invalidUrls.add("https://www.youtube.com/"); + for(String invalidUrl: invalidUrls) { + Throwable exception = null; + try { + urlIdHandler.getId(invalidUrl); + } catch (ParsingException e) { + exception = e; + } + if(exception == null) { + fail("Expected ParsingException for url: " + invalidUrl); + } + } + } + @Test + public void getId() throws Exception { + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI")); + assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s")); + assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY")); + assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s")); + assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8")); + assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI")); + assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI")); + + // Shared links + String sharedId = "7JIArTByb3E"; + String realId = "Q7JsK50NGaA"; + assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); + assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId )); + assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId)); + } + + + @Test + public void testAcceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); + assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); + assertTrue(urlIdHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100")); + assertTrue(urlIdHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s")); + //assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch/jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); + assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); + assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); + + assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); + + String sharedId = "8A940MXKFmQ"; + assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); + assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId )); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId)); + } +} \ No newline at end of file diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java new file mode 100644 index 0000000000..c47828ab40 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java @@ -0,0 +1,51 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.schabi.newpipe.extractor.ServiceList.Youtube; + +/* + * Created by Christian Schabesberger on 18.11.16. + * + * Copyright (C) Christian Schabesberger 2016 + * YoutubeSuggestionExtractorTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * Test for {@link SuggestionExtractor} + */ +public class YoutubeSuggestionExtractorTest { + private SuggestionExtractor suggestionExtractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + suggestionExtractor = Youtube.getService().getSuggestionExtractor(); + } + + @Test + public void testIfSuggestions() throws IOException, ExtractionException { + assertFalse(suggestionExtractor.suggestionList("hello", "de").isEmpty()); + } +}