From 82b072a0819e03d934cb22c12d1877a206989b4f Mon Sep 17 00:00:00 2001 From: SpiritCroc Date: Tue, 2 Aug 2022 14:54:27 +0200 Subject: [PATCH] Generate missing video thumbnails locally Change-Id: I72b71cf1de2ae2922494cb08c8c9baa607b11562 --- .../de/spiritcroc/util/ThumbnailExtractor.kt | 104 ++++++++++++++++++ .../app/core/glide/VectorGlideModelLoader.kt | 15 ++- .../timeline/factory/MessageItemFactory.kt | 5 +- .../image/ImageContentRendererFactory.kt | 5 +- .../features/media/ImageContentRenderer.kt | 10 +- 5 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 vector/src/main/java/de/spiritcroc/util/ThumbnailExtractor.kt diff --git a/vector/src/main/java/de/spiritcroc/util/ThumbnailExtractor.kt b/vector/src/main/java/de/spiritcroc/util/ThumbnailExtractor.kt new file mode 100644 index 0000000000..5ec2bfbebc --- /dev/null +++ b/vector/src/main/java/de/spiritcroc/util/ThumbnailExtractor.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package de.spiritcroc.util + +import android.content.Context +import android.graphics.Bitmap +import android.media.MediaMetadataRetriever +import android.net.Uri +import timber.log.Timber +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.FileOutputStream +import java.lang.Exception +import javax.inject.Inject + +/** + * Based on org.matrix.android.sdk.internal.session.content.ThumbnailExtractor, but more useful for + * rendering video thumbnails locally (instead of the SDK's focus on uploading thumbnails). + * I.e. we re-use existing MediaData classes, and keep the SDK's class package-private. + */ +class ThumbnailExtractor @Inject constructor( + private val context: Context +) { + + companion object { + private const val DEBUG_THUMBNAIL_EXTRACTOR = true + } + + + fun extractThumbnail(file: File): Result? { + // In case we want to generate thumbnails for non-video files here as well, we need to fix below MIME-type detection. + // Currently, it returns false for isMimeTypeVideo on mp4 videos + /* + val type = MimeTypeMap.getFileExtensionFromUrl(file.absolutePath) + if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.v("MIME type is $type: isVideo: ${type.isMimeTypeVideo()}") + return if (type.isMimeTypeVideo()) { + extractVideoThumbnail(file) + } else { + null + } + */ + // Currently, only video thumbnail generation supported + return extractVideoThumbnail(file) + } + + private fun getThumbnailCacheFile(videoFile: File): File { + val thumbnailCacheDir = File(context.cacheDir, "localThumbnails") + return File(thumbnailCacheDir, "${videoFile.canonicalPath}.jpg") + } + + private fun extractVideoThumbnail(file: File): Result { + val thumbnailFile = getThumbnailCacheFile(file) + if (thumbnailFile.exists()) { + if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.d("Return cached thumbnail ${thumbnailFile.canonicalPath}") + return Result.success(thumbnailFile) + } + if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.d("Generate thumbnail ${thumbnailFile.canonicalPath}") + val mediaMetadataRetriever = MediaMetadataRetriever() + try { + mediaMetadataRetriever.setDataSource(context, Uri.fromFile(file)) + mediaMetadataRetriever.frameAtTime?.let { thumbnail -> + val outputStream = ByteArrayOutputStream() + thumbnail.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) + /* + val thumbnailWidth = thumbnail.width + val thumbnailHeight = thumbnail.height + val thumbnailSize = outputStream.size() + */ + val tmpFile = File(thumbnailFile.parentFile, "${file.name}.part") + tmpFile.parentFile?.mkdirs() + FileOutputStream(tmpFile).use { + it.write(outputStream.toByteArray()) + } + tmpFile.renameTo(thumbnailFile) + thumbnail.recycle() + outputStream.reset() + } ?: run { + Timber.e("Cannot extract video thumbnail at %s", file.canonicalPath) + return Result.failure(Exception("Cannot extract video thumbnail at ${file.canonicalPath}")) + } + } catch (e: Exception) { + Timber.e(e, "Cannot extract video thumbnail") + return Result.failure(e) + } finally { + mediaMetadataRetriever.release() + } + return Result.success(thumbnailFile) + } + +} diff --git a/vector/src/main/java/im/vector/app/core/glide/VectorGlideModelLoader.kt b/vector/src/main/java/im/vector/app/core/glide/VectorGlideModelLoader.kt index 92c95d3062..2d3429c309 100644 --- a/vector/src/main/java/im/vector/app/core/glide/VectorGlideModelLoader.kt +++ b/vector/src/main/java/im/vector/app/core/glide/VectorGlideModelLoader.kt @@ -25,6 +25,7 @@ import com.bumptech.glide.load.model.ModelLoader import com.bumptech.glide.load.model.ModelLoaderFactory import com.bumptech.glide.load.model.MultiModelLoaderFactory import com.bumptech.glide.signature.ObjectKey +import de.spiritcroc.util.ThumbnailExtractor import im.vector.app.core.extensions.singletonEntryPoint import im.vector.app.core.files.LocalFilesHelper import im.vector.app.features.media.ImageContentRenderer @@ -71,6 +72,8 @@ class VectorGlideDataFetcher( private val localFilesHelper = LocalFilesHelper(context) private val activeSessionHolder = context.singletonEntryPoint().activeSessionHolder() + private val thumbnailExtractor = ThumbnailExtractor(context) + private val client = activeSessionHolder.getSafeActiveSession()?.getOkHttpClient() ?: OkHttpClient() override fun getDataClass(): Class { @@ -118,7 +121,7 @@ class VectorGlideDataFetcher( } // Use the file vector service, will avoid flickering and redownload after upload activeSessionHolder.getSafeActiveSession()?.coroutineScope?.launch { - val result = runCatching { + var result = runCatching { fileService.downloadFile( fileName = data.filename, mimeType = data.mimeType, @@ -126,6 +129,16 @@ class VectorGlideDataFetcher( elementToDecrypt = data.elementToDecrypt ) } + if (result.isFailure && (data.fallbackUrl != null || data.fallbackElementToDecrypt != null)) { + result = runCatching { + fileService.downloadFile( + fileName = data.filename, + mimeType = data.mimeType, + url = data.fallbackUrl, + elementToDecrypt = data.fallbackElementToDecrypt) + } + result = result.getOrNull()?.let { thumbnailExtractor.extractThumbnail(it) } ?: result + } withContext(Dispatchers.Main) { result.fold( { callback.onDataReady(it.inputStream()) }, diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/MessageItemFactory.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/MessageItemFactory.kt index 167b4b3a22..646a1595ec 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/MessageItemFactory.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/MessageItemFactory.kt @@ -498,7 +498,10 @@ class MessageItemFactory @Inject constructor( maxHeight = maxHeight, width = messageContent.videoInfo?.width, maxWidth = maxWidth, - allowNonMxcUrls = informationData.sendState.isSending() + allowNonMxcUrls = informationData.sendState.isSending(), + // Video fallback for generating thumbnails + fallbackUrl = messageContent.getFileUrl(), + fallbackElementToDecrypt = messageContent.encryptedFileInfo?.toElementToDecrypt() ) val videoData = VideoContentRenderer.Data( diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/image/ImageContentRendererFactory.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/image/ImageContentRendererFactory.kt index 91bbd584e1..71d996d2ec 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/image/ImageContentRendererFactory.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/image/ImageContentRendererFactory.kt @@ -57,7 +57,10 @@ fun TimelineEvent.buildImageContentRendererData(maxHeight: Int): ImageContentRen maxHeight = maxHeight, width = videoInfo?.thumbnailInfo?.width, maxWidth = maxHeight * 2, - allowNonMxcUrls = false + allowNonMxcUrls = false, + // Video fallback for generating thumbnails + fallbackUrl = messageVideoContent.getFileUrl(), + fallbackElementToDecrypt = messageVideoContent.encryptedFileInfo?.toElementToDecrypt() ) } else -> null diff --git a/vector/src/main/java/im/vector/app/features/media/ImageContentRenderer.kt b/vector/src/main/java/im/vector/app/features/media/ImageContentRenderer.kt index cf0af25d5d..2ad4f83d39 100644 --- a/vector/src/main/java/im/vector/app/features/media/ImageContentRenderer.kt +++ b/vector/src/main/java/im/vector/app/features/media/ImageContentRenderer.kt @@ -83,7 +83,10 @@ class ImageContentRenderer @Inject constructor( val width: Int?, val maxWidth: Int, // If true will load non mxc url, be careful to set it only for images sent by you - override val allowNonMxcUrls: Boolean = false + override val allowNonMxcUrls: Boolean = false, + // Fallback for videos: generate preview from video + val fallbackUrl: String? = null, + val fallbackElementToDecrypt: ElementToDecrypt? = null, ) : AttachmentData enum class Mode { @@ -159,6 +162,7 @@ class ImageContentRenderer @Inject constructor( var request = createGlideRequest(data, mode, imageView, size) .listener(object : RequestListener { override fun onLoadFailed(e: GlideException?, model: Any?, target: Target?, isFirstResource: Boolean): Boolean { + Timber.e(e, "Glide image render failed") return false } @@ -268,8 +272,8 @@ class ImageContentRenderer @Inject constructor( } fun createGlideRequest(data: Data, mode: Mode, glideRequests: GlideRequests, size: Size = processSize(data, mode)): GlideRequest { - return if (data.elementToDecrypt != null) { - // Encrypted image + return if (data.elementToDecrypt != null || (data.url == null && data.fallbackUrl != null)) { + // Encrypted image, or video without thumbnail url glideRequests .load(data) .diskCacheStrategy(DiskCacheStrategy.NONE)