From 287d1dfd63d8bacec02f53724309374798c45dfc Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sun, 29 May 2022 19:08:18 +0200 Subject: [PATCH] [SoundCloud] Use the HLS delivery method for all streams and extract only a single stream URL from HLS manifest for MP3 streams SoundCloud broke the workaround used to get a single file from HLS manifests for Opus manifests, but it still works for MP3 ones. The code has been adapted to prevent an unneeded request (the one to the Opus HLS manifest) and the HLS delivery method is now used for SoundCloud MP3 and Opus streams, plus the progressive one (for tracks which have a progressive stream (MP3) and for the ones which doesn't have one, it is still used by trying to get a progressive stream, using the workaround). Streams extraction has been also moved to Java 8 Stream's API and the relevant test has been also updated. --- .../extractors/SoundcloudStreamExtractor.java | 175 ++++++++++-------- .../SoundcloudStreamExtractorTest.java | 16 +- 2 files changed, 109 insertions(+), 82 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index bacfd077e1..42a832cded 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -28,6 +28,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -169,7 +170,6 @@ public List getAudioStreams() throws ExtractionException { // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. - // If audio streams were calculated, return the calculated result if (!track.getBoolean("streamable") || !isAvailable) { return audioStreams; } @@ -181,36 +181,29 @@ public List getAudioStreams() throws ExtractionException { extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings), audioStreams); } + extractDownloadableFileIfAvailable(audioStreams); } catch (final NullPointerException e) { - throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e); + throw new ExtractionException("Could not get audio streams", e); } return audioStreams; } private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) { - boolean presence = false; - for (final Object transcoding : transcodings) { - final JsonObject transcodingJsonObject = (JsonObject) transcoding; - if (transcodingJsonObject.getString("preset").contains("mp3") - && transcodingJsonObject.getObject("format").getString("protocol") - .equals("progressive")) { - presence = true; - break; - } - } - return presence; + return transcodings.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset") + .contains("mp3") && transcodingJsonObject.getObject("format") + .getString("protocol").equals("progressive")); } @Nonnull - private String getTranscodingUrl(final String endpointUrl, - final String protocol) + private String getTranscodingUrl(final String endpointUrl) throws IOException, ExtractionException { - final Downloader downloader = NewPipe.getDownloader(); - final String apiStreamUrl = endpointUrl + "?client_id=" - + clientId(); - final String response = downloader.get(apiStreamUrl).responseBody(); + final String apiStreamUrl = endpointUrl + "?client_id=" + clientId(); + final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody(); final JsonObject urlObject; try { urlObject = JsonParser.object().from(response); @@ -218,16 +211,7 @@ private String getTranscodingUrl(final String endpointUrl, throw new ParsingException("Could not parse streamable URL", e); } - final String urlString = urlObject.getString("url"); - - if (protocol.equals("progressive")) { - return urlString; - } else if (protocol.equals("hls")) { - return getSingleUrlFromHlsManifest(urlString); - } - - // else, unknown protocol - return EMPTY_STRING; + return urlObject.getString("url"); } @Nullable @@ -252,50 +236,87 @@ private String getDownloadUrl(@Nonnull final String trackId) private void extractAudioStreams(@Nonnull final JsonArray transcodings, final boolean mp3ProgressiveInStreams, final List audioStreams) { - for (final Object transcoding : transcodings) { - final JsonObject transcodingJsonObject = (JsonObject) transcoding; - final String url = transcodingJsonObject.getString("url"); - if (isNullOrEmpty(url)) { - continue; - } - - final String mediaUrl; - final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN); - final String protocol = transcodingJsonObject.getObject("format") - .getString("protocol"); - MediaFormat mediaFormat = null; - int averageBitrate = UNKNOWN_BITRATE; - if (preset.contains("mp3")) { - // Don't add the MP3 HLS stream if there is a progressive stream present - // because the two have the same bitrate - if (mp3ProgressiveInStreams && protocol.equals("hls")) { - continue; - } - mediaFormat = MediaFormat.MP3; - averageBitrate = 128; - } else if (preset.contains("opus")) { - mediaFormat = MediaFormat.OPUS; - averageBitrate = 64; - } + transcodings.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .forEachOrdered(transcoding -> { + final String url = transcoding.getString("url"); + if (isNullOrEmpty(url)) { + return; + } - try { - mediaUrl = getTranscodingUrl(url, protocol); - if (!mediaUrl.isEmpty()) { - final AudioStream audioStream = new AudioStream.Builder() - .setId(preset) - .setContent(mediaUrl, true) - .setMediaFormat(mediaFormat) - .setAverageBitrate(averageBitrate) - .build(); - if (!Stream.containSimilarStream(audioStream, audioStreams)) { - audioStreams.add(audioStream); + final String preset = transcoding.getString("preset", ID_UNKNOWN); + final String protocol = transcoding.getObject("format").getString("protocol"); + final AudioStream.Builder builder = new AudioStream.Builder() + .setId(preset); + + try { + // streamUrl can be either the MP3 progressive stream URL or the + // manifest URL of the HLS MP3 stream (if there is no MP3 progressive + // stream, see above) + final String streamUrl = getTranscodingUrl(url); + + if (preset.contains("mp3")) { + // Don't add the MP3 HLS stream if there is a progressive stream + // present because the two have the same bitrate + final boolean isHls = protocol.equals("hls"); + if (mp3ProgressiveInStreams && isHls) { + return; + } + + builder.setMediaFormat(MediaFormat.MP3); + builder.setAverageBitrate(128); + + if (isHls) { + builder.setDeliveryMethod(DeliveryMethod.HLS); + builder.setContent(streamUrl, true); + + final AudioStream hlsStream = builder.build(); + if (!Stream.containSimilarStream(hlsStream, audioStreams)) { + audioStreams.add(hlsStream); + } + + final String progressiveHlsUrl = + getSingleUrlFromHlsManifest(streamUrl); + builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP); + builder.setContent(progressiveHlsUrl, true); + + final AudioStream progressiveHlsStream = builder.build(); + if (!Stream.containSimilarStream( + progressiveHlsStream, audioStreams)) { + audioStreams.add(progressiveHlsStream); + } + + // The MP3 HLS stream has been added in both versions (HLS and + // progressive with the manifest parsing trick), so we need to + // continue (otherwise the code would try to add again the stream, + // which would be not added because the containsSimilarStream + // method would return false and an audio stream object would be + // created for nothing) + return; + } else { + builder.setContent(streamUrl, true); + } + } else if (preset.contains("opus")) { + // The HLS manifest trick doesn't work for opus streams + builder.setContent(streamUrl, true); + builder.setMediaFormat(MediaFormat.OPUS); + builder.setAverageBitrate(64); + builder.setDeliveryMethod(DeliveryMethod.HLS); + } else { + // Unknown format, skip to the next audio stream + return; + } + + final AudioStream audioStream = builder.build(); + if (!Stream.containSimilarStream(audioStream, audioStreams)) { + audioStreams.add(audioStream); + } + } catch (final ExtractionException | IOException ignored) { + // Something went wrong when trying to get and add this audio stream, + // skip to the next one } - } - } catch (final Exception ignored) { - // Something went wrong when parsing this transcoding URL, so don't add it to the - // audioStreams - } - } + }); } /** @@ -332,7 +353,7 @@ public void extractDownloadableFileIfAvailable(final List audioStre } /** - * Parses a SoundCloud HLS manifest to get a single URL of HLS streams. + * Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams. * *

* This method downloads the provided manifest URL, finds all web occurrences in the manifest, @@ -340,17 +361,20 @@ public void extractDownloadableFileIfAvailable(final List audioStre * this as a string. *

* + *

+ * This was working before for Opus streams, but has been broken by SoundCloud. + *

+ * * @param hlsManifestUrl the URL of the manifest to be parsed * @return a single URL that contains a range equal to the length of the track */ @Nonnull private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl) throws ParsingException { - final Downloader dl = NewPipe.getDownloader(); final String hlsManifestResponse; try { - hlsManifestResponse = dl.get(hlsManifestUrl).responseBody(); + hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody(); } catch (final IOException | ReCaptchaException e) { throw new ParsingException("Could not get SoundCloud HLS manifest"); } @@ -359,12 +383,13 @@ private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManif for (int l = lines.length - 1; l >= 0; l--) { final String line = lines[l]; // Get the last URL from manifest, because it contains the range of the stream - if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) { + if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) { final String[] hlsLastRangeUrlArray = line.split("/"); return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/" + hlsLastRangeUrlArray[6]; } } + throw new ParsingException("Could not get any URL from HLS manifest"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index b72f054dbb..da2c0511dd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -188,25 +188,27 @@ public void testAudioStreams() throws Exception { super.testAudioStreams(); final List audioStreams = extractor.getAudioStreams(); assertEquals(2, audioStreams.size()); - for (final AudioStream audioStream : audioStreams) { + audioStreams.forEach(audioStream -> { final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); - assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); final String mediaUrl = audioStream.getContent(); if (audioStream.getFormat() == MediaFormat.OPUS) { // Assert that it's an OPUS 64 kbps media URL with a single range which comes // from an HLS SoundCloud CDN ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); ExtractorAsserts.assertContains(".64.opus", mediaUrl); - } - if (audioStream.getFormat() == MediaFormat.MP3) { + assertSame(DeliveryMethod.HLS, deliveryMethod, + "Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + } else if (audioStream.getFormat() == MediaFormat.MP3) { // Assert that it's a MP3 128 kbps media URL which comes from a progressive // SoundCloud CDN ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", mediaUrl); + assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod, + "Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); } - } + }); } } }