diff --git a/app/build.gradle.kts b/app/build.gradle.kts index eca981a350..ef74dcb2f8 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -127,6 +127,9 @@ dependencies { implementation(libs.kotlinx.datetime) implementation(libs.kotlinx.serialization.retrofit) + /* NewPipe Extractor */ + implementation(libs.newpipeextractor) + /* Cronet and Coil */ coreLibraryDesugaring(libs.desugaring) implementation(libs.cronet.embedded) diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index d77cd4094e..b6211ddeb4 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -114,3 +114,9 @@ # Settings fragments are loaded through reflection -keep class com.github.libretube.ui.preferences.** { *; } + +## Rules for NewPipeExtractor +-keep class org.schabi.newpipe.extractor.timeago.patterns.** { *; } +-keep class org.mozilla.javascript.** { *; } +-keep class org.mozilla.classfile.ClassFileWriter +-dontwarn org.mozilla.javascript.tools.** diff --git a/app/src/main/java/com/github/libretube/api/PlaylistsHelper.kt b/app/src/main/java/com/github/libretube/api/PlaylistsHelper.kt index 3cce0d4d5e..d3ca927e77 100644 --- a/app/src/main/java/com/github/libretube/api/PlaylistsHelper.kt +++ b/app/src/main/java/com/github/libretube/api/PlaylistsHelper.kt @@ -191,7 +191,7 @@ object PlaylistsHelper { MAX_CONCURRENT_IMPORT_CALLS ).map { videos -> videos.parallelMap { - runCatching { RetrofitInstance.api.getStreams(it) } + runCatching { StreamsExtractor.extractStreams(it) } .getOrNull() ?.toStreamItem(it) }.filterNotNull() diff --git a/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt b/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt new file mode 100644 index 0000000000..97c7897e01 --- /dev/null +++ b/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt @@ -0,0 +1,108 @@ +package com.github.libretube.api + +import com.github.libretube.api.obj.ChapterSegment +import com.github.libretube.api.obj.MetaInfo +import com.github.libretube.api.obj.PreviewFrames +import com.github.libretube.api.obj.StreamItem +import com.github.libretube.api.obj.Streams +import com.github.libretube.api.obj.Subtitle +import com.github.libretube.helpers.PlayerHelper +import com.github.libretube.util.NewPipeDownloaderImpl +import kotlinx.datetime.toKotlinInstant +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.stream.StreamInfo +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +object StreamsExtractor { +// val npe by lazy { +// NewPipe.getService(ServiceList.YouTube.serviceId) +// } + + init { + NewPipe.init(NewPipeDownloaderImpl()) + } + + suspend fun extractStreams(videoId: String): Streams { + if (!PlayerHelper.disablePipedProxy || !PlayerHelper.localStreamExtraction) { + return RetrofitInstance.api.getStreams(videoId) + } + + val resp = StreamInfo.getInfo("https://www.youtube.com/watch?v=$videoId") + return Streams( + title = resp.name, + description = resp.description.toString(), + uploader = resp.uploaderName, + uploaderAvatar = resp.uploaderAvatars.maxBy { it.height }.url, + uploaderUrl = resp.uploaderUrl, + uploaderVerified = resp.isUploaderVerified, + uploaderSubscriberCount = resp.uploaderSubscriberCount, + category = resp.category, + views = resp.viewCount, + likes = resp.likeCount, + license = resp.licence, + hls = resp.hlsUrl, + dash = resp.dashMpdUrl, + tags = resp.tags, + metaInfo = resp.metaInfo.map { + MetaInfo( + it.title, + it.content.content, + it.urls.map { url -> url.toString() }, + it.urlTexts + ) + }, + visibility = resp.privacy.name.lowercase(), + duration = resp.duration, + uploadTimestamp = resp.uploadDate.offsetDateTime().toInstant().toKotlinInstant(), + uploaded = resp.uploadDate.offsetDateTime().toEpochSecond(), + thumbnailUrl = resp.thumbnails.maxBy { it.height }.url, + relatedStreams = resp.relatedItems.map { it as StreamInfoItem }.map { + StreamItem( + it.url, + it.infoType.name, + it.name, + it.thumbnails.maxBy { image -> image.height }.url, + it.uploaderName, + it.uploaderUrl, + it.uploaderAvatars.maxBy { image -> image.height }.url, + it.textualUploadDate, + it.duration, + it.viewCount, + it.isUploaderVerified, + it.uploadDate?.offsetDateTime()?.toEpochSecond() ?: 0L, + it.shortDescription, + it.isShortFormContent, + ) + }, + chapters = resp.streamSegments.map { + ChapterSegment( + title = it.title, + image = it.previewUrl.orEmpty(), + start = it.startTimeSeconds.toLong() + ) + }, + audioStreams = emptyList(), // TODO: audio streams and video streams via DASH, currently broken anyways + videoStreams = emptyList(), + previewFrames = resp.previewFrames.map { + PreviewFrames( + it.urls, + it.frameWidth, + it.frameHeight, + it.totalCount, + it.durationPerFrame.toLong(), + it.framesPerPageX, + it.framesPerPageY + ) + }, + subtitles = resp.subtitles.map { + Subtitle( + it.content, + it.format?.mimeType, + it.displayLanguageName, + it.languageTag, + it.isAutoGenerated + ) + } + ) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/github/libretube/constants/PreferenceKeys.kt b/app/src/main/java/com/github/libretube/constants/PreferenceKeys.kt index 6e168aa0f0..7f0ae36741 100644 --- a/app/src/main/java/com/github/libretube/constants/PreferenceKeys.kt +++ b/app/src/main/java/com/github/libretube/constants/PreferenceKeys.kt @@ -128,6 +128,7 @@ object PreferenceKeys { const val MAX_CONCURRENT_DOWNLOADS = "max_parallel_downloads" const val EXTERNAL_DOWNLOAD_PROVIDER = "external_download_provider" const val DISABLE_VIDEO_IMAGE_PROXY = "disable_video_image_proxy" + const val LOCAL_STREAM_EXTRACTION = "local_stream_extraction" // History const val WATCH_HISTORY_SIZE = "watch_history_size" diff --git a/app/src/main/java/com/github/libretube/helpers/PlayerHelper.kt b/app/src/main/java/com/github/libretube/helpers/PlayerHelper.kt index 6a628338fc..40a762c9b1 100644 --- a/app/src/main/java/com/github/libretube/helpers/PlayerHelper.kt +++ b/app/src/main/java/com/github/libretube/helpers/PlayerHelper.kt @@ -350,6 +350,12 @@ object PlayerHelper { false ) + val localStreamExtraction: Boolean + get() = PreferenceHelper.getBoolean( + PreferenceKeys.LOCAL_STREAM_EXTRACTION, + true + ) + val useHlsOverDash: Boolean get() = PreferenceHelper.getBoolean( PreferenceKeys.USE_HLS_OVER_DASH, diff --git a/app/src/main/java/com/github/libretube/helpers/ProxyHelper.kt b/app/src/main/java/com/github/libretube/helpers/ProxyHelper.kt index f2657b351f..f521dcac55 100644 --- a/app/src/main/java/com/github/libretube/helpers/ProxyHelper.kt +++ b/app/src/main/java/com/github/libretube/helpers/ProxyHelper.kt @@ -36,7 +36,7 @@ object ProxyHelper { * Detect whether the proxy should be used or not for a given stream URL based on user preferences */ fun unwrapStreamUrl(url: String): String { - return if (PlayerHelper.disablePipedProxy) { + return if (PlayerHelper.disablePipedProxy && !PlayerHelper.localStreamExtraction) { unwrapUrl(url) } else { url diff --git a/app/src/main/java/com/github/libretube/services/DownloadService.kt b/app/src/main/java/com/github/libretube/services/DownloadService.kt index adef62d056..50387151f2 100644 --- a/app/src/main/java/com/github/libretube/services/DownloadService.kt +++ b/app/src/main/java/com/github/libretube/services/DownloadService.kt @@ -20,6 +20,7 @@ import com.github.libretube.LibreTubeApp.Companion.DOWNLOAD_CHANNEL_NAME import com.github.libretube.R import com.github.libretube.api.CronetHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.constants.IntentData import com.github.libretube.db.DatabaseHolder.Database import com.github.libretube.db.obj.Download @@ -109,7 +110,7 @@ class DownloadService : LifecycleService() { lifecycleScope.launch(coroutineContext) { try { val streams = withContext(Dispatchers.IO) { - RetrofitInstance.api.getStreams(videoId) + StreamsExtractor.extractStreams(videoId) } val thumbnailTargetPath = getDownloadPath(DownloadHelper.THUMBNAIL_DIR, fileName) @@ -386,7 +387,7 @@ class DownloadService : LifecycleService() { * Regenerate stream url using available info format and quality. */ private suspend fun regenerateLink(item: DownloadItem) { - val streams = RetrofitInstance.api.getStreams(item.videoId) + val streams = StreamsExtractor.extractStreams(item.videoId) val stream = when (item.type) { FileType.AUDIO -> streams.audioStreams FileType.VIDEO -> streams.videoStreams diff --git a/app/src/main/java/com/github/libretube/services/OnlinePlayerService.kt b/app/src/main/java/com/github/libretube/services/OnlinePlayerService.kt index 6412cc27f3..3d4cbe98c9 100644 --- a/app/src/main/java/com/github/libretube/services/OnlinePlayerService.kt +++ b/app/src/main/java/com/github/libretube/services/OnlinePlayerService.kt @@ -28,6 +28,7 @@ import com.github.libretube.LibreTubeApp.Companion.PLAYER_CHANNEL_NAME import com.github.libretube.R import com.github.libretube.api.JsonHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.Segment import com.github.libretube.api.obj.Streams import com.github.libretube.constants.IntentData @@ -253,7 +254,7 @@ class OnlinePlayerService : LifecycleService() { lifecycleScope.launch(Dispatchers.IO) { streams = runCatching { - RetrofitInstance.api.getStreams(videoId) + StreamsExtractor.extractStreams(videoId) }.getOrNull() ?: return@launch // clear the queue if it shouldn't be kept explicitly diff --git a/app/src/main/java/com/github/libretube/services/PlaylistDownloadEnqueueService.kt b/app/src/main/java/com/github/libretube/services/PlaylistDownloadEnqueueService.kt index 2fa6b2b4ed..1cef399701 100644 --- a/app/src/main/java/com/github/libretube/services/PlaylistDownloadEnqueueService.kt +++ b/app/src/main/java/com/github/libretube/services/PlaylistDownloadEnqueueService.kt @@ -12,6 +12,7 @@ import com.github.libretube.LibreTubeApp.Companion.PLAYLIST_DOWNLOAD_ENQUEUE_CHA import com.github.libretube.R import com.github.libretube.api.PlaylistsHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.PipedStream import com.github.libretube.api.obj.StreamItem import com.github.libretube.constants.IntentData @@ -136,7 +137,7 @@ class PlaylistDownloadEnqueueService : LifecycleService() { for (stream in streams) { val videoInfo = runCatching { - RetrofitInstance.api.getStreams(stream.url!!.toID()) + StreamsExtractor.extractStreams(stream.url!!.toID()) }.getOrNull() ?: continue val videoStream = getStream(videoInfo.videoStreams, maxVideoQuality) diff --git a/app/src/main/java/com/github/libretube/ui/dialogs/AddToPlaylistDialog.kt b/app/src/main/java/com/github/libretube/ui/dialogs/AddToPlaylistDialog.kt index 23473db1e1..d9f5840c3f 100644 --- a/app/src/main/java/com/github/libretube/ui/dialogs/AddToPlaylistDialog.kt +++ b/app/src/main/java/com/github/libretube/ui/dialogs/AddToPlaylistDialog.kt @@ -16,6 +16,7 @@ import androidx.lifecycle.repeatOnLifecycle import com.github.libretube.R import com.github.libretube.api.PlaylistsHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.Playlists import com.github.libretube.constants.IntentData import com.github.libretube.databinding.DialogAddToPlaylistBinding @@ -112,7 +113,7 @@ class AddToPlaylistDialog : DialogFragment() { val streams = when { videoId != null -> listOfNotNull( runCatching { - RetrofitInstance.api.getStreams(videoId!!).toStreamItem(videoId!!) + StreamsExtractor.extractStreams(videoId!!).toStreamItem(videoId!!) }.getOrNull() ) diff --git a/app/src/main/java/com/github/libretube/ui/dialogs/DownloadDialog.kt b/app/src/main/java/com/github/libretube/ui/dialogs/DownloadDialog.kt index e79d5b7ba3..540a7604f0 100644 --- a/app/src/main/java/com/github/libretube/ui/dialogs/DownloadDialog.kt +++ b/app/src/main/java/com/github/libretube/ui/dialogs/DownloadDialog.kt @@ -14,6 +14,7 @@ import androidx.fragment.app.setFragmentResult import androidx.lifecycle.lifecycleScope import com.github.libretube.R import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.PipedStream import com.github.libretube.api.obj.Streams import com.github.libretube.api.obj.Subtitle @@ -81,7 +82,7 @@ class DownloadDialog : DialogFragment() { lifecycleScope.launch { val response = try { withContext(Dispatchers.IO) { - RetrofitInstance.api.getStreams(videoId) + StreamsExtractor.extractStreams(videoId) } } catch (e: IOException) { println(e) diff --git a/app/src/main/java/com/github/libretube/ui/models/PlayerViewModel.kt b/app/src/main/java/com/github/libretube/ui/models/PlayerViewModel.kt index a3a9459dc6..826088bc2f 100644 --- a/app/src/main/java/com/github/libretube/ui/models/PlayerViewModel.kt +++ b/app/src/main/java/com/github/libretube/ui/models/PlayerViewModel.kt @@ -10,6 +10,7 @@ import androidx.media3.exoplayer.trackselection.DefaultTrackSelector import com.github.libretube.R import com.github.libretube.api.JsonHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.Message import com.github.libretube.api.obj.Segment import com.github.libretube.api.obj.Streams @@ -57,7 +58,7 @@ class PlayerViewModel : ViewModel() { if (isOrientationChangeInProgress && streamsInfo != null) return@withContext streamsInfo to null streamsInfo = try { - RetrofitInstance.api.getStreams(videoId).deArrow(videoId) + StreamsExtractor.extractStreams(videoId).deArrow(videoId) } catch (e: IOException) { return@withContext null to context.getString(R.string.unknown_error) } catch (e: HttpException) { diff --git a/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt b/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt new file mode 100644 index 0000000000..b648f831b6 --- /dev/null +++ b/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt @@ -0,0 +1,60 @@ +package com.github.libretube.util + +import java.io.IOException +import java.util.concurrent.TimeUnit +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.downloader.Request +import org.schabi.newpipe.extractor.downloader.Response +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException + +class NewPipeDownloaderImpl : Downloader() { + private val client: OkHttpClient = OkHttpClient.Builder() + .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .build() + + @Throws(IOException::class, ReCaptchaException::class) + override fun execute(request: Request): Response { + val url = request.url() + + val requestBody = request.dataToSend()?.let { + it.toRequestBody(APPLICATION_JSON, 0, it.size) + } + + val requestBuilder = okhttp3.Request.Builder() + .method(request.httpMethod(), requestBody) + .url(url) + .addHeader(USER_AGENT_HEADER_NAME, USER_AGENT) + + for ((headerName, headerValueList) in request.headers()) { + requestBuilder.removeHeader(headerName) + for (headerValue in headerValueList) { + requestBuilder.addHeader(headerName, headerValue) + } + } + + val response = client.newCall(requestBuilder.build()).execute() + if (response.code == CAPTCHA_STATUS_CODE) { + response.close() + throw ReCaptchaException("reCaptcha Challenge requested", url) + } + + return Response( + response.code, + response.message, + response.headers.toMultimap(), + response.body?.string(), + response.request.url.toString() + ) + } + + companion object { + private const val USER_AGENT_HEADER_NAME = "User-Agent" + private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0" + private const val CAPTCHA_STATUS_CODE = 429 + private val APPLICATION_JSON = "application/json".toMediaType() + private const val READ_TIMEOUT_SECONDS = 30L + } +} \ No newline at end of file diff --git a/app/src/main/java/com/github/libretube/util/PlayingQueue.kt b/app/src/main/java/com/github/libretube/util/PlayingQueue.kt index ac59b5581f..4d859d89e5 100644 --- a/app/src/main/java/com/github/libretube/util/PlayingQueue.kt +++ b/app/src/main/java/com/github/libretube/util/PlayingQueue.kt @@ -4,6 +4,7 @@ import android.util.Log import androidx.media3.common.Player import com.github.libretube.api.PlaylistsHelper import com.github.libretube.api.RetrofitInstance +import com.github.libretube.api.StreamsExtractor import com.github.libretube.api.obj.StreamItem import com.github.libretube.extensions.move import com.github.libretube.extensions.runCatchingIO @@ -179,7 +180,7 @@ object PlayingQueue { } fun insertByVideoId(videoId: String) = runCatchingIO { - val streams = RetrofitInstance.api.getStreams(videoId.toID()) + val streams = StreamsExtractor.extractStreams(videoId.toID()) add(streams.toStreamItem(videoId)) } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f5859c342f..f5bb2cf3e7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -510,6 +510,8 @@ Gestures External download provider Enter the package name of the app you want to use for downloading videos. Leave blank to use LibreTube\'s inbuilt downloader. + Local stream extraction + Directly fetch video playback information from YouTube without using Piped. Download Service diff --git a/app/src/main/res/xml/instance_settings.xml b/app/src/main/res/xml/instance_settings.xml index 4f9401e5f4..578e36d759 100644 --- a/app/src/main/res/xml/instance_settings.xml +++ b/app/src/main/res/xml/instance_settings.xml @@ -73,6 +73,14 @@ android:title="@string/disable_proxy" app:key="disable_video_image_proxy" /> + + \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 1a73258c5b..eb4aa51c18 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,6 +20,7 @@ allprojects { repositories { google() mavenCentral() + maven { setUrl("https://jitpack.io") } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index eda3418749..1b8a27cd5e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,7 @@ constraintlayout = "2.1.4" loggingInterceptor = "4.12.0" material = "1.12.0" navigation = "2.7.7" +newpipeextractor = "6e3a4a6d9de61eafb73e7eb1b714847b5077856d" preference = "1.2.1" extJunit = "1.2.1" espresso = "3.6.1" @@ -62,6 +63,7 @@ androidx-media3-exoplayer-dash = { group = "androidx.media3", name="media3-exopl androidx-media3-datasource-cronet = { group = "androidx.media3", name = "media3-datasource-cronet", version.ref = "media3" } androidx-media3-session = { group="androidx.media3", name="media3-session", version.ref="media3" } androidx-media3-ui = { group="androidx.media3", name="media3-ui", version.ref="media3" } +newpipeextractor = { module = "com.github.TeamNewPipe:NewPipeExtractor", version.ref = "newpipeextractor" } square-retrofit = { group = "com.squareup.retrofit2", name = "retrofit", version.ref = "retrofit" } desugaring = { group = "com.android.tools", name = "desugar_jdk_libs_nio", version.ref = "desugaring" } cronet-embedded = { group = "org.chromium.net", name = "cronet-embedded", version.ref = "cronetEmbedded" }