From 1b41b9563a8faa557638b608f1d1449f33546328 Mon Sep 17 00:00:00 2001 From: Andrew Gunnerson Date: Thu, 19 Dec 2024 19:36:22 -0500 Subject: [PATCH] Add support for DocumentsProvider.openDocumentThumbnail() This will return a pipe and then spawn a background task to generate a thumbnail and write it to the pipe. Cancellation is supported in between logical operations (eg. parse headers, load frame, etc.). Android has no way to interrupt the parsers in MediaMetadataRetriever and ImageDecoder though, so there will be a delay before cancellation takes effect. This can be noticeable when scrolling quickly, for example through a list of videos, since there is a hard concurrency limit equal to the number of logical cores. Note that only audio, image, and video formats natively supported by Android can be thumbnailed. We don't bundle any decoders. Signed-off-by: Andrew Gunnerson --- app/build.gradle.kts | 1 + .../chiller3/rsaf/rclone/RcloneProvider.kt | 96 +++++- .../com/chiller3/rsaf/rclone/Thumbnailer.kt | 320 ++++++++++++++++++ gradle/libs.versions.toml | 2 + gradle/verification-metadata.xml | 8 + 5 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 app/src/main/java/com/chiller3/rsaf/rclone/Thumbnailer.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 96ed870..98cdfaf 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -196,6 +196,7 @@ dependencies { implementation(libs.appcompat) implementation(libs.biometric) implementation(libs.core.ktx) + implementation(libs.exifinterface) implementation(libs.fragment.ktx) implementation(libs.preference.ktx) implementation(libs.security.crypto) diff --git a/app/src/main/java/com/chiller3/rsaf/rclone/RcloneProvider.kt b/app/src/main/java/com/chiller3/rsaf/rclone/RcloneProvider.kt index 38afecd..773c476 100644 --- a/app/src/main/java/com/chiller3/rsaf/rclone/RcloneProvider.kt +++ b/app/src/main/java/com/chiller3/rsaf/rclone/RcloneProvider.kt @@ -11,6 +11,7 @@ import android.content.SharedPreferences import android.content.res.AssetFileDescriptor import android.database.Cursor import android.database.MatrixCursor +import android.graphics.Bitmap import android.graphics.Point import android.os.CancellationSignal import android.os.Handler @@ -24,6 +25,7 @@ import android.system.ErrnoException import android.system.Os import android.system.OsConstants import android.util.Log +import android.util.Size import android.webkit.MimeTypeMap import com.chiller3.rsaf.AppLock import com.chiller3.rsaf.BuildConfig @@ -39,6 +41,8 @@ import com.chiller3.rsaf.extension.toException import com.chiller3.rsaf.extension.toSingleLineString import java.io.FileNotFoundException import java.io.IOException +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreferenceChangeListener { companion object { @@ -76,7 +80,8 @@ class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreference DocumentsContract.Document.FLAG_SUPPORTS_MOVE or DocumentsContract.Document.FLAG_SUPPORTS_REMOVE or DocumentsContract.Document.FLAG_SUPPORTS_RENAME or - DocumentsContract.Document.FLAG_SUPPORTS_WRITE + DocumentsContract.Document.FLAG_SUPPORTS_WRITE or + DocumentsContract.Document.FLAG_SUPPORTS_THUMBNAIL private val DIRECTORY_PERMS = OsConstants.S_IRWXU or OsConstants.S_IRGRP or OsConstants.S_IXGRP or @@ -310,6 +315,8 @@ class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreference // consistent. It is just a dumb workaround to make some common file access patterns work. A // client app can absolutely still shoot itself in the foot. private val inUseTracker = VfsNode() + private val thumbnailTaskPool = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()) private fun waitUntilUploadsDone(documentId: String) { val path = vfsPath(documentId) @@ -382,6 +389,9 @@ class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreference override fun shutdown() { prefs.unregisterListener(this) + + thumbnailTaskPool.shutdown() + thumbnailTaskPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS) } override fun onSharedPreferenceChanged(sharedPreferences: SharedPreferences?, key: String?) { @@ -579,16 +589,46 @@ class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreference /** * Open a document thumbnail. * - * This is not implemented. It's only overridden because certain clients, including DocumentsUI, - * would otherwise crash, even though [DocumentsContract.Document.FLAG_SUPPORTS_THUMBNAIL] is - * never advertised. + * This is only implemented for audio, image, and video files. If generating a thumbnail is + * supported, but the process fails, the client will see an empty file. */ override fun openDocumentThumbnail(documentId: String, sizeHint: Point, signal: CancellationSignal?): AssetFileDescriptor? { debugLog("openDocumentThumbnail($documentId, $sizeHint, $signal)") enforceNotBlocked(documentId) - return null + val projection = arrayOf(DocumentsContract.Document.COLUMN_MIME_TYPE) + val mimeType = queryDocument(documentId, projection).use { cursor -> + if (!cursor.moveToFirst()) { + // Should never happen. + return null + } + + val index = cursor.getColumnIndexOrThrow(DocumentsContract.Document.COLUMN_MIME_TYPE) + cursor.getString(index) + } + + if (!Thumbnailer.isSupported(mimeType)) { + Log.d(TAG, "Thumbnail not supported for: $mimeType") + return null + } + + val mediaInput = openDocument(documentId, "r", signal) + + try { + val pipe = ParcelFileDescriptor.createReliablePipe() + + // The task owns both file descriptors. + val task = ThumbnailTask(documentId, mediaInput, pipe[1], mimeType, sizeHint, signal) + + thumbnailTaskPool.submit(task) + + return AssetFileDescriptor(pipe[0], 0, AssetFileDescriptor.UNKNOWN_LENGTH) + } catch (e: Exception) { + Log.w(TAG, "Failed to create thumbnail pipe", e) + mediaInput.close() + return null + } } /** @@ -1002,4 +1042,48 @@ class RcloneProvider : DocumentsProvider(), SharedPreferences.OnSharedPreference toString(this, 0) } } -} \ No newline at end of file + + private inner class ThumbnailTask( + private val documentId: String, + private val mediaInput: ParcelFileDescriptor, + private val thumbnailOutput: ParcelFileDescriptor, + private val mimeType: String, + private val sizeHint: Point, + private val signal: CancellationSignal?, + ) : Runnable { + override fun run() { + debugLog("ThumbnailTask[$documentId].run()") + + // Since we're accessing the data through ProxyFd, we don't need to try and keep the + // process alive during this process. + + mediaInput.use { input -> + // We'll try to close with an error message if possible, in case the client is able + // to make use of that. There's no other way of indicating an error or that a + // thumbnail is unavailable. A client that doesn't check for errors will just see an + // empty file. + ParcelFileDescriptor.AutoCloseOutputStream(thumbnailOutput).use { output -> + try { + val bitmap = Thumbnailer.createThumbnail( + input.fileDescriptor, + mimeType, + Size(sizeHint.x, sizeHint.y), + signal, + ) + + try { + signal?.throwIfCanceled() + + bitmap.compress(Bitmap.CompressFormat.PNG, 0, output) + } finally { + bitmap.recycle() + } + } catch (e: Exception) { + Log.w(TAG, "Failed to generate thumbnail", e) + thumbnailOutput.closeWithError(e.toSingleLineString()) + } + } + } + } + } +} diff --git a/app/src/main/java/com/chiller3/rsaf/rclone/Thumbnailer.kt b/app/src/main/java/com/chiller3/rsaf/rclone/Thumbnailer.kt new file mode 100644 index 0000000..4c3bd5c --- /dev/null +++ b/app/src/main/java/com/chiller3/rsaf/rclone/Thumbnailer.kt @@ -0,0 +1,320 @@ +/* + * SPDX-FileCopyrightText: 2024 Andrew Gunnerson + * SPDX-License-Identifier: GPL-3.0-only + */ + +package com.chiller3.rsaf.rclone + +import android.content.res.AssetFileDescriptor +import android.graphics.Bitmap +import android.graphics.ImageDecoder +import android.graphics.Matrix +import android.media.MediaMetadataRetriever +import android.media.ThumbnailUtils +import android.os.Build +import android.os.CancellationSignal +import android.os.ParcelFileDescriptor +import android.system.Os +import android.system.OsConstants +import android.util.Size +import androidx.exifinterface.media.ExifInterface +import java.io.FileDescriptor +import java.io.FileNotFoundException +import java.io.IOException +import java.nio.ByteBuffer +import kotlin.math.max + +/** + * Helper functions for creating thumbnails of audio, image, and video files. It is similar to + * [ThumbnailUtils], except: + * + * * the source is a file descriptor + * * no filesystem traversals are performed + * * there are more cancellation points + * * EXIF image flip orientation tag values are supported + * * the more capable [ExifInterface] androidx library is used + */ +object Thumbnailer { + private const val PRE_Q_MEM_SIZE_LIMIT = 8L * 1024 * 1024 + + /** Set desired image scale before decoding to reduce memory usage. */ + private class SubsampleDuringDecode( + private val size: Size, + private val signal: CancellationSignal?, + ) : ImageDecoder.OnHeaderDecodedListener { + override fun onHeaderDecoded( + decoder: ImageDecoder, + info: ImageDecoder.ImageInfo, + source: ImageDecoder.Source, + ) { + signal?.throwIfCanceled() + + decoder.setAllocator(ImageDecoder.ALLOCATOR_SOFTWARE) + + val widthScale = info.size.width / size.width + val heightScale = info.size.height / size.height + val inverseScale = max(widthScale, heightScale) + if (inverseScale > 1) { + decoder.setTargetSampleSize(inverseScale) + } + } + } + + /** Check whether thumbnail generation is theoretically supported for a MIME type. */ + fun isSupported(mimeType: String): Boolean = + mimeType.startsWith("audio/") + || mimeType.startsWith("image/") + || mimeType.startsWith("video/") + + /** Check whether an image MIME type is stored as one or more frames in a video container. */ + private fun isImageInVideoContainer(mimeType: String): Boolean = + mimeType == "image/avif" + || mimeType == "image/heic" + || mimeType == "image/heic-sequence" + || mimeType == "image/heif" + || mimeType == "image/heif-sequence" + + /** + * Get the embedded thumbnail from an audio file. Throws [FileNotFoundException] if the file + * does not have an embedded thumbnail. + */ + private fun createAudioThumbnail( + fd: FileDescriptor, + sizeHint: Size, + signal: CancellationSignal?, + ): Bitmap { + signal?.throwIfCanceled() + + val retriever = MediaMetadataRetriever() + + try { + retriever.setDataSource(fd) + + val embedded = retriever.embeddedPicture + ?: throw FileNotFoundException("Audio file has no embedded thumbnail") + + signal?.throwIfCanceled() + + val source = ImageDecoder.createSource(ByteBuffer.wrap(embedded)) + val subsampler = SubsampleDuringDecode(sizeHint, signal) + + return ImageDecoder.decodeBitmap(source, subsampler) + } finally { + retriever.release() + } + + throw FileNotFoundException("Audio file has no embedded thumbnail") + } + + /** + * Get the embedded thumbnail from an image file. If the file does not have one, then the image + * itself is used to generate a thumbnail. On Android P, this fallback path requires reading the + * entire file into memory, so an [IOException] will be thrown if the file size exceeds a limit. + * If the image file format is stored in a video container, then the first image in the video + * container is used to create the thumbnail. + */ + private fun createImageThumbnail( + fd: FileDescriptor, + mimeType: String, + sizeHint: Size, + signal: CancellationSignal?, + ): Bitmap { + var bitmap: Bitmap? = null + + // Try to extract the embedded thumbnail from image formats stored in video containers. + if (isImageInVideoContainer(mimeType)) { + signal?.throwIfCanceled() + + val retriever = MediaMetadataRetriever() + + try { + retriever.setDataSource(fd) + + // getThumbnailImageAtIndex() is not on the hidden API whitelist, so we'll have to + // use getImageAtIndex() instead and do the downscaling ourselves. + val image = retriever.getImageAtIndex(-1) + if (image != null) { + signal?.throwIfCanceled() + + bitmap = ThumbnailUtils.extractThumbnail(image, sizeHint.width, sizeHint.height) + if (bitmap !== image) { + image.recycle() + } + } + } finally { + retriever.release() + } + } + + signal?.throwIfCanceled() + + // This dup()s the fd and uses its own copy when parsing the file. For HEIF, this will + // internally parse the file again with MediaMetadataRetriever. There is no way to prevent + // this from happening. + val exif = ExifInterface(fd) + val orientation = exif.getAttributeInt( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_UNDEFINED, + ) + + val (rotation, scaleX, scaleY) = when (orientation) { + ExifInterface.ORIENTATION_UNDEFINED, ExifInterface.ORIENTATION_NORMAL -> Triple(0, 1, 1) + ExifInterface.ORIENTATION_ROTATE_90 -> Triple(90, 1, 1) + ExifInterface.ORIENTATION_ROTATE_180 -> Triple(180, 1, 1) + ExifInterface.ORIENTATION_ROTATE_270 -> Triple(270, 1, 1) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> Triple(0, -1, 1) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> Triple(0, 1, -1) + ExifInterface.ORIENTATION_TRANSPOSE -> Triple(90, -1, 0) + ExifInterface.ORIENTATION_TRANSVERSE -> Triple(270, -1, 0) + else -> throw IllegalStateException("Invalid orientation tag value: $orientation") + } + + val subsampler = SubsampleDuringDecode(sizeHint, signal) + + // Try to extract the thumbnail embedded in the EXIF data. + if (bitmap == null) { + val thumbnail = exif.thumbnailBytes + if (thumbnail != null) { + signal?.throwIfCanceled() + + val source = ImageDecoder.createSource(ByteBuffer.wrap(thumbnail)) + + try { + bitmap = ImageDecoder.decodeBitmap(source, subsampler) + } catch (_: ImageDecoder.DecodeException) { + // Ignore + } + } + } + + // We need to manually apply rotation and X/Y flips since the EXIF orientation tag is not + // processed when loading embedded thumbnails. + if (bitmap != null) { + val needRotation = rotation != 0 + val needScale = scaleX != 1 || scaleY != 1 + + if (needRotation || needScale) { + signal?.throwIfCanceled() + + val centerX = (bitmap.width / 2).toFloat() + val centerY = (bitmap.height / 2).toFloat() + val matrix = Matrix() + + if (needRotation) { + matrix.setRotate(rotation.toFloat(), centerX, centerY) + } + if (needScale) { + matrix.postScale(scaleX.toFloat(), scaleY.toFloat(), centerX, centerY) + } + + val newBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, false) + if (newBitmap !== bitmap) { + bitmap.recycle() + } + + bitmap = newBitmap + } + } + + // If there is no embedded thumbnail, then we'll create one from the image. This will + // automatically process the EXIF orientation tag. + if (bitmap == null) { + signal?.throwIfCanceled() + + val fileSize = Os.lseek(fd, 0, OsConstants.SEEK_END) + + val source = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + ImageDecoder.createSource { + AssetFileDescriptor(ParcelFileDescriptor.dup(fd), 0, fileSize) + } + } else if (fileSize <= PRE_Q_MEM_SIZE_LIMIT) { + // Android P has no way to read from an arbitrary file descriptor, so we'll have to + // read things into memory. We'll only do this if the file is small. + val buffer = ByteBuffer.allocate(fileSize.toInt()) + + val n = Os.pread(fd, buffer, 0) + if (n != buffer.limit()) { + throw IOException("Unexpected EOF: $n != $fileSize") + } + + ImageDecoder.createSource(buffer) + } else { + throw IOException("Image too large to read into memory: $fileSize") + } + + bitmap = ImageDecoder.decodeBitmap(source, subsampler) + } + + return bitmap + } + + /** + * Get the embedded thumbnail from a video file. If the file does not have one, then a thumbnail + * is created from the closest keyframe at the half point of the video. + */ + private fun createVideoThumbnail( + fd: FileDescriptor, + sizeHint: Size, + signal: CancellationSignal?, + ): Bitmap { + signal?.throwIfCanceled() + + val retriever = MediaMetadataRetriever() + + try { + retriever.setDataSource(fd) + + val embedded = retriever.embeddedPicture + if (embedded != null) { + signal?.throwIfCanceled() + + val source = ImageDecoder.createSource(ByteBuffer.wrap(embedded)) + val subsampler = SubsampleDuringDecode(sizeHint, signal) + + return ImageDecoder.decodeBitmap(source, subsampler) + } + + val width = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) + ?.toInt() ?: throw IOException("Failed to get video width") + val height = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) + ?.toInt() ?: throw IOException("Failed to get video height") + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLong() ?: throw IOException("Failed to get video duration") + + val thumbnailUs = durationMs * 1000 / 2 + val option = MediaMetadataRetriever.OPTION_CLOSEST_SYNC + + signal?.throwIfCanceled() + + return if (sizeHint.width >= width && sizeHint.height >= height) { + retriever.getFrameAtTime(thumbnailUs, option) + } else { + retriever.getScaledFrameAtTime(thumbnailUs, option, sizeHint.width, sizeHint.height) + } ?: throw IOException("Failed to extract frame from video") + } finally { + retriever.release() + } + } + + /** + * Get the thumbnail of an audio, image, or video file. Throws [IllegalArgumentException] if + * [mimeType] does not refer to an audio, image, or video MIME type. + */ + fun createThumbnail( + fd: FileDescriptor, + mimeType: String, + sizeHint: Size, + signal: CancellationSignal?, + ): Bitmap { + return if (mimeType.startsWith("audio/")) { + createAudioThumbnail(fd, sizeHint, signal) + } else if (mimeType.startsWith("image/")) { + createImageThumbnail(fd, mimeType, sizeHint, signal) + } else if (mimeType.startsWith("video/")) { + createVideoThumbnail(fd, sizeHint, signal) + } else { + throw IllegalArgumentException("Unsupported MIME type: $mimeType") + } + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cf7c0f2..5d4b503 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,7 @@ biometric = "1.1.0" core-ktx = "1.15.0" jgit = "7.1.0.202411261347-r" espresso-core = "3.6.1" +exifinterface = "1.3.7" fragment-ktx = "1.8.5" junit = "1.2.1" kotlin = "2.1.0" @@ -20,6 +21,7 @@ appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "a biometric = { group = "androidx.biometric", name = "biometric", version.ref = "biometric" } core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "core-ktx" } espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espresso-core" } +exifinterface = { group = "androidx.exifinterface", name = "exifinterface", version.ref = "exifinterface" } fragment-ktx = { group = "androidx.fragment", name = "fragment-ktx", version.ref = "fragment-ktx" } jgit = { group = "org.eclipse.jgit", name = "org.eclipse.jgit", version.ref = "jgit" } jgit-archive = { group = "org.eclipse.jgit", name = "org.eclipse.jgit.archive", version.ref = "jgit" } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 7c7f13f..3465ec5 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -330,6 +330,14 @@ + + + + + + + +