Skip to content

Commit

Permalink
[SoundCloud] Use the HLS delivery method for all streams and extract …
Browse files Browse the repository at this point in the history 
…only a single stream URL from HLS manifest for MP3 streams

SoundCloud broke the workaround used to get a single file from HLS manifests for Opus manifests, but it still works for MP3 ones.

The code has been adapted to prevent an unneeded request (the one to the Opus HLS manifest) and the HLS delivery method is now used for SoundCloud MP3 and Opus streams, plus the progressive one (for tracks which have a progressive stream (MP3) and for the ones which doesn't have one, it is still used by trying to get a progressive stream, using the workaround).

Streams extraction has been also moved to Java 8 Stream's API and the relevant test has been also updated.
  • Loading branch information
@AudricV
AudricV committed May 29, 2022
1 parent b3c620f commit 287d1df
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 82 deletions.
175 changes: 100 additions & 75 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
Expand Down Expand Up @@ -169,7 +170,6 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
// Streams can be streamable and downloadable - or explicitly not.
// For playing the track, it is only necessary to have a streamable track.
// If this is not the case, this track might not be published yet.
// If audio streams were calculated, return the calculated result
if (!track.getBoolean("streamable") || !isAvailable) {
return audioStreams;
}
Expand All @@ -181,53 +181,37 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
audioStreams);
}

extractDownloadableFileIfAvailable(audioStreams);
} catch (final NullPointerException e) {
throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
throw new ExtractionException("Could not get audio streams", e);
}

return audioStreams;
}

private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
boolean presence = false;
for (final Object transcoding : transcodings) {
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
if (transcodingJsonObject.getString("preset").contains("mp3")
&& transcodingJsonObject.getObject("format").getString("protocol")
.equals("progressive")) {
presence = true;
break;
}
}
return presence;
return transcodings.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
.contains("mp3") && transcodingJsonObject.getObject("format")
.getString("protocol").equals("progressive"));
}

@Nonnull
private String getTranscodingUrl(final String endpointUrl,
final String protocol)
private String getTranscodingUrl(final String endpointUrl)
throws IOException, ExtractionException {
final Downloader downloader = NewPipe.getDownloader();
final String apiStreamUrl = endpointUrl + "?client_id="
+ clientId();
final String response = downloader.get(apiStreamUrl).responseBody();
final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
final JsonObject urlObject;
try {
urlObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse streamable URL", e);
}

final String urlString = urlObject.getString("url");

if (protocol.equals("progressive")) {
return urlString;
} else if (protocol.equals("hls")) {
return getSingleUrlFromHlsManifest(urlString);
}

// else, unknown protocol
return EMPTY_STRING;
return urlObject.getString("url");
}

@Nullable
Expand All @@ -252,50 +236,87 @@ private String getDownloadUrl(@Nonnull final String trackId)
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
final boolean mp3ProgressiveInStreams,
final List<AudioStream> audioStreams) {
for (final Object transcoding : transcodings) {
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
final String url = transcodingJsonObject.getString("url");
if (isNullOrEmpty(url)) {
continue;
}

final String mediaUrl;
final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN);
final String protocol = transcodingJsonObject.getObject("format")
.getString("protocol");
MediaFormat mediaFormat = null;
int averageBitrate = UNKNOWN_BITRATE;
if (preset.contains("mp3")) {
// Don't add the MP3 HLS stream if there is a progressive stream present
// because the two have the same bitrate
if (mp3ProgressiveInStreams && protocol.equals("hls")) {
continue;
}
mediaFormat = MediaFormat.MP3;
averageBitrate = 128;
} else if (preset.contains("opus")) {
mediaFormat = MediaFormat.OPUS;
averageBitrate = 64;
}
transcodings.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.forEachOrdered(transcoding -> {
final String url = transcoding.getString("url");
if (isNullOrEmpty(url)) {
return;
}

try {
mediaUrl = getTranscodingUrl(url, protocol);
if (!mediaUrl.isEmpty()) {
final AudioStream audioStream = new AudioStream.Builder()
.setId(preset)
.setContent(mediaUrl, true)
.setMediaFormat(mediaFormat)
.setAverageBitrate(averageBitrate)
.build();
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
final String preset = transcoding.getString("preset", ID_UNKNOWN);
final String protocol = transcoding.getObject("format").getString("protocol");
final AudioStream.Builder builder = new AudioStream.Builder()
.setId(preset);

try {
// streamUrl can be either the MP3 progressive stream URL or the
// manifest URL of the HLS MP3 stream (if there is no MP3 progressive
// stream, see above)
final String streamUrl = getTranscodingUrl(url);

if (preset.contains("mp3")) {
// Don't add the MP3 HLS stream if there is a progressive stream
// present because the two have the same bitrate
final boolean isHls = protocol.equals("hls");
if (mp3ProgressiveInStreams && isHls) {
return;
}

builder.setMediaFormat(MediaFormat.MP3);
builder.setAverageBitrate(128);

if (isHls) {
builder.setDeliveryMethod(DeliveryMethod.HLS);
builder.setContent(streamUrl, true);

final AudioStream hlsStream = builder.build();
if (!Stream.containSimilarStream(hlsStream, audioStreams)) {
audioStreams.add(hlsStream);
}

final String progressiveHlsUrl =
getSingleUrlFromHlsManifest(streamUrl);
builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP);
builder.setContent(progressiveHlsUrl, true);

final AudioStream progressiveHlsStream = builder.build();
if (!Stream.containSimilarStream(
progressiveHlsStream, audioStreams)) {
audioStreams.add(progressiveHlsStream);
}

// The MP3 HLS stream has been added in both versions (HLS and
// progressive with the manifest parsing trick), so we need to
// continue (otherwise the code would try to add again the stream,
// which would be not added because the containsSimilarStream
// method would return false and an audio stream object would be
// created for nothing)
return;
} else {
builder.setContent(streamUrl, true);
}
} else if (preset.contains("opus")) {
// The HLS manifest trick doesn't work for opus streams
builder.setContent(streamUrl, true);
builder.setMediaFormat(MediaFormat.OPUS);
builder.setAverageBitrate(64);
builder.setDeliveryMethod(DeliveryMethod.HLS);
} else {
// Unknown format, skip to the next audio stream
return;
}

final AudioStream audioStream = builder.build();
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
} catch (final ExtractionException | IOException ignored) {
// Something went wrong when trying to get and add this audio stream,
// skip to the next one
}
}
} catch (final Exception ignored) {
// Something went wrong when parsing this transcoding URL, so don't add it to the
// audioStreams
}
}
});
}

/**
Expand Down Expand Up @@ -332,25 +353,28 @@ public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStre
}

/**
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
* Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams.
*
* <p>
* This method downloads the provided manifest URL, finds all web occurrences in the manifest,
* gets the last segment URL, changes its segment range to {@code 0/track-length}, and return
* this as a string.
* </p>
*
* <p>
* This was working before for Opus streams, but has been broken by SoundCloud.
* </p>
*
* @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track
*/
@Nonnull
private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
throws ParsingException {
final Downloader dl = NewPipe.getDownloader();
final String hlsManifestResponse;

try {
hlsManifestResponse = dl.get(hlsManifestUrl).responseBody();
hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody();
} catch (final IOException | ReCaptchaException e) {
throw new ParsingException("Could not get SoundCloud HLS manifest");
}
Expand All @@ -359,12 +383,13 @@ private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManif
for (int l = lines.length - 1; l >= 0; l--) {
final String line = lines[l];
// Get the last URL from manifest, because it contains the range of the stream
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) {
final String[] hlsLastRangeUrlArray = line.split("/");
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
+ "/" + hlsLastRangeUrlArray[6];
}
}

throw new ParsingException("Could not get any URL from HLS manifest");
}

Expand Down
16 changes: 9 additions & 7 deletions .../java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,25 +188,27 @@ public void testAudioStreams() throws Exception {
super.testAudioStreams();
final List<AudioStream> audioStreams = extractor.getAudioStreams();
assertEquals(2, audioStreams.size());
for (final AudioStream audioStream : audioStreams) {
audioStreams.forEach(audioStream -> {
final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
"Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
final String mediaUrl = audioStream.getContent();
if (audioStream.getFormat() == MediaFormat.OPUS) {
// Assert that it's an OPUS 64 kbps media URL with a single range which comes
// from an HLS SoundCloud CDN
ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
ExtractorAsserts.assertContains(".64.opus", mediaUrl);
}
if (audioStream.getFormat() == MediaFormat.MP3) {
assertSame(DeliveryMethod.HLS, deliveryMethod,
"Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
} else if (audioStream.getFormat() == MediaFormat.MP3) {
// Assert that it's a MP3 128 kbps media URL which comes from a progressive
// SoundCloud CDN
ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
mediaUrl);
assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
"Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
}
}
});
}
}
}

0 comments on commit 287d1df

Please sign in to comment.