Timeline : extract video thumbnail and upload it

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentAttachmentData.kt

@@ -27,8 +27,8 @@ data class ContentAttachmentData(
         val height: Long? = 0,
         val width: Long? = 0,
         val name: String? = null,
-        val path: String? = null,
-        val mimeType: String? = null,
+        val path: String,
+        val mimeType: String,
         val type: Type
 ) : Parcelable {
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentUploadStateTracker.kt

@@ -18,9 +18,15 @@ package im.vector.matrix.android.api.session.content
 
 interface ContentUploadStateTracker {
 
-    fun track(eventId: String, updateListener: UpdateListener)
+    fun track(key: String, updateListener: UpdateListener)
 
-    fun untrack(eventId: String, updateListener: UpdateListener)
+    fun untrack(key: String, updateListener: UpdateListener)
+
+    fun setFailure(key: String)
+
+    fun setSuccess(key: String)
+
+    fun setProgress(key: String, current: Long, total: Long)
 
     interface UpdateListener {
         fun onUpdate(state: State)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentModule.kt

@@ -31,7 +31,7 @@ internal class ContentModule {
         }
 
         scope(DefaultSession.SCOPE) {
-            ContentUploader(get(), get(), get<ContentUploadStateTracker>() as DefaultContentUploadStateTracker)
+            FileUploader(get(), get())
         }
 
         scope(DefaultSession.SCOPE) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/DefaultContentUploadStateTracker.kt

@@ -23,42 +23,42 @@ import im.vector.matrix.android.api.session.content.ContentUploadStateTracker
 internal class DefaultContentUploadStateTracker : ContentUploadStateTracker {
 
     private val mainHandler = Handler(Looper.getMainLooper())
-    private val progressByEvent = mutableMapOf<String, ContentUploadStateTracker.State>()
-    private val listenersByEvent = mutableMapOf<String, MutableList<ContentUploadStateTracker.UpdateListener>>()
+    private val states = mutableMapOf<String, ContentUploadStateTracker.State>()
+    private val listeners = mutableMapOf<String, MutableList<ContentUploadStateTracker.UpdateListener>>()
 
-    override fun track(eventId: String, updateListener: ContentUploadStateTracker.UpdateListener) {
-        val listeners = listenersByEvent[eventId] ?: ArrayList()
+    override fun track(key: String, updateListener: ContentUploadStateTracker.UpdateListener) {
+        val listeners = listeners[key] ?: ArrayList()
         listeners.add(updateListener)
-        listenersByEvent[eventId] = listeners
-        val currentState = progressByEvent[eventId] ?: ContentUploadStateTracker.State.Idle
+        this.listeners[key] = listeners
+        val currentState = states[key] ?: ContentUploadStateTracker.State.Idle
         mainHandler.post { updateListener.onUpdate(currentState) }
     }
 
-    override fun untrack(eventId: String, updateListener: ContentUploadStateTracker.UpdateListener) {
-        listenersByEvent[eventId]?.apply {
+    override fun untrack(key: String, updateListener: ContentUploadStateTracker.UpdateListener) {
+        listeners[key]?.apply {
             remove(updateListener)
         }
     }
 
-    internal fun setFailure(eventId: String) {
+    override fun setFailure(key: String) {
         val failure = ContentUploadStateTracker.State.Failure
-        updateState(eventId, failure)
+        updateState(key, failure)
     }
 
-    internal fun setSuccess(eventId: String) {
+    override fun setSuccess(key: String) {
         val success = ContentUploadStateTracker.State.Success
-        updateState(eventId, success)
+        updateState(key, success)
     }
 
-    internal fun setProgress(eventId: String, current: Long, total: Long) {
+    override fun setProgress(key: String, current: Long, total: Long) {
         val progressData = ContentUploadStateTracker.State.ProgressData(current, total)
-        updateState(eventId, progressData)
+        updateState(key, progressData)
     }
 
-    private fun updateState(eventId: String, state: ContentUploadStateTracker.State) {
-        progressByEvent[eventId] = state
+    private fun updateState(key: String, state: ContentUploadStateTracker.State) {
+        states[key] = state
         mainHandler.post {
-            listenersByEvent[eventId]?.also { listeners ->
+            listeners[key]?.also { listeners ->
                 listeners.forEach { it.onUpdate(state) }
             }
         }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/FileUploader.kt

@@ -19,7 +19,6 @@ package im.vector.matrix.android.internal.session.content
 import arrow.core.Try
 import arrow.core.Try.Companion.raise
 import im.vector.matrix.android.api.auth.data.SessionParams
-import im.vector.matrix.android.api.session.content.ContentAttachmentData
 import im.vector.matrix.android.internal.di.MoshiProvider
 import im.vector.matrix.android.internal.network.ProgressRequestBody
 import okhttp3.HttpUrl
@@ -31,44 +30,51 @@ import java.io.File
 import java.io.IOException
 
 
-internal class ContentUploader(private val okHttpClient: OkHttpClient,
-                               private val sessionParams: SessionParams,
-                               private val contentUploadProgressTracker: DefaultContentUploadStateTracker) {
+internal class FileUploader(private val okHttpClient: OkHttpClient,
+                            private val sessionParams: SessionParams) {
+
+    private val uploadUrl = sessionParams.homeServerConnectionConfig.homeServerUri.toString() + URI_PREFIX_CONTENT_API + "upload"
 
     private val moshi = MoshiProvider.providesMoshi()
     private val responseAdapter = moshi.adapter(ContentUploadResponse::class.java)
 
-    fun uploadFile(eventId: String, attachment: ContentAttachmentData): Try<ContentUploadResponse> {
-        if (attachment.path == null || attachment.mimeType == null) {
-            return raise(RuntimeException())
-        }
-        val file = File(attachment.path)
-        val urlString = sessionParams.homeServerConnectionConfig.homeServerUri.toString() + URI_PREFIX_CONTENT_API + "upload"
 
-        val urlBuilder = HttpUrl.parse(urlString)?.newBuilder()
-                         ?: return raise(RuntimeException())
+    fun uploadFile(file: File,
+                   filename: String?,
+                   mimeType: String,
+                   progressListener: ProgressRequestBody.Listener? = null): Try<ContentUploadResponse> {
+
+        val uploadBody = RequestBody.create(MediaType.parse(mimeType), file)
+        return upload(uploadBody, filename, progressListener)
+
+    }
+
+    fun uploadByteArray(byteArray: ByteArray,
+                        filename: String?,
+                        mimeType: String,
+                        progressListener: ProgressRequestBody.Listener? = null): Try<ContentUploadResponse> {
+
+        val uploadBody = RequestBody.create(MediaType.parse(mimeType), byteArray)
+        return upload(uploadBody, filename, progressListener)
+
+    }
+
+
+    private fun upload(uploadBody: RequestBody, filename: String?, progressListener: ProgressRequestBody.Listener?): Try<ContentUploadResponse> {
+        val urlBuilder = HttpUrl.parse(uploadUrl)?.newBuilder() ?: return raise(RuntimeException())
 
         val httpUrl = urlBuilder
-                .addQueryParameter(
-                        "filename", attachment.name
-                ).build()
+                .addQueryParameter("filename", filename)
+                .build()
 
-        val requestBody = RequestBody.create(
-                MediaType.parse(attachment.mimeType),
-                file
-        )
-        val progressRequestBody = ProgressRequestBody(requestBody, object : ProgressRequestBody.Listener {
-            override fun onProgress(current: Long, total: Long) {
-                contentUploadProgressTracker.setProgress(eventId, current, total)
-            }
-        })
+        val requestBody = if (progressListener != null) ProgressRequestBody(uploadBody, progressListener) else uploadBody
 
         val request = Request.Builder()
                 .url(httpUrl)
-                .post(progressRequestBody)
+                .post(requestBody)
                 .build()
 
-        val result = Try {
+        return Try {
             okHttpClient.newCall(request).execute().use { response ->
                 if (!response.isSuccessful) {
                     throw IOException()
@@ -80,11 +86,7 @@ internal class ContentUploader(private val okHttpClient: OkHttpClient,
                 }
             }
         }
-        if (result.isFailure()) {
-            contentUploadProgressTracker.setFailure(eventId)
-        } else {
-            contentUploadProgressTracker.setSuccess(eventId)
-        }
-        return result
+
     }
+
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ThumbnailExtractor.kt

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.session.content
+
+import android.graphics.Bitmap
+import android.media.ThumbnailUtils
+import android.provider.MediaStore
+import im.vector.matrix.android.api.session.content.ContentAttachmentData
+import java.io.ByteArrayOutputStream
+import java.io.File
+
+internal object ThumbnailExtractor {
+
+    class ThumbnailData(
+            val width: Int,
+            val height: Int,
+            val size: Long,
+            val bytes: ByteArray,
+            val mimeType: String
+    )
+
+    fun extractThumbnail(attachment: ContentAttachmentData): ThumbnailData? {
+        val file = File(attachment.path)
+        if (!file.exists() || !file.isFile) {
+            return null
+        }
+        return if (attachment.type == ContentAttachmentData.Type.VIDEO) {
+            extractVideoThumbnail(attachment)
+        } else {
+            null
+        }
+    }
+
+    private fun extractVideoThumbnail(attachment: ContentAttachmentData): ThumbnailData? {
+        val thumbnail = ThumbnailUtils.createVideoThumbnail(attachment.path, MediaStore.Video.Thumbnails.MINI_KIND)
+        val outputStream = ByteArrayOutputStream()
+        thumbnail.compress(Bitmap.CompressFormat.JPEG, 100, outputStream)
+        val thumbnailWidth = thumbnail.width
+        val thumbnailHeight = thumbnail.height
+        val thumbnailSize = outputStream.size()
+        val thumbnailData = ThumbnailData(
+                width = thumbnailWidth,
+                height = thumbnailHeight,
+                size = thumbnailSize.toLong(),
+                bytes = outputStream.toByteArray(),
+                mimeType = "image/jpeg"
+        )
+        thumbnail.recycle()
+        outputStream.reset()
+        return thumbnailData
+    }
+
+
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/UploadContentWorker.kt

@@ -21,6 +21,7 @@ import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import im.vector.matrix.android.api.session.content.ContentAttachmentData
+import im.vector.matrix.android.api.session.content.ContentUploadStateTracker
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.events.model.toContent
 import im.vector.matrix.android.api.session.events.model.toModel
@@ -30,14 +31,19 @@ import im.vector.matrix.android.api.session.room.model.message.MessageFileConten
 import im.vector.matrix.android.api.session.room.model.message.MessageImageContent
 import im.vector.matrix.android.api.session.room.model.message.MessageVideoContent
 import im.vector.matrix.android.internal.di.MatrixKoinComponent
+import im.vector.matrix.android.internal.network.ProgressRequestBody
 import im.vector.matrix.android.internal.session.room.send.SendEventWorker
 import im.vector.matrix.android.internal.util.WorkerParamsFactory
 import org.koin.standalone.inject
+import timber.log.Timber
+import java.io.File
+
 
 internal class UploadContentWorker(context: Context, params: WorkerParameters)
     : CoroutineWorker(context, params), MatrixKoinComponent {
 
-    private val mediaUploader by inject<ContentUploader>()
+    private val fileUploader by inject<FileUploader>()
+    private val contentUploadProgressTracker by inject<ContentUploadStateTracker>()
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
@@ -50,29 +56,63 @@ internal class UploadContentWorker(context: Context, params: WorkerParameters)
         val params = WorkerParamsFactory.fromData<Params>(inputData)
                      ?: return Result.failure()
 
-        if (params.event.eventId == null) {
-            return Result.failure()
+        val eventId = params.event.eventId ?: return Result.failure()
+        val attachment = params.attachment
+
+        val thumbnailData = ThumbnailExtractor.extractThumbnail(params.attachment)
+        val attachmentFile = createAttachmentFile(attachment) ?: return Result.failure()
+        var uploadedThumbnailUrl: String? = null
+
+        if (thumbnailData != null) {
+            fileUploader
+                    .uploadByteArray(thumbnailData.bytes, "thumb_${attachment.name}", thumbnailData.mimeType)
+                    .fold(
+                            { Timber.e(it) },
+                            { uploadedThumbnailUrl = it.contentUri }
+                    )
         }
-        return mediaUploader
-                .uploadFile(params.event.eventId, params.attachment)
-                .fold({ handleFailure() }, { handleSuccess(params, it) })
+
+        val progressListener = object : ProgressRequestBody.Listener {
+            override fun onProgress(current: Long, total: Long) {
+                contentUploadProgressTracker.setProgress(eventId, current, total)
+            }
+        }
+        return fileUploader
+                .uploadFile(attachmentFile, attachment.name, attachment.mimeType, progressListener)
+                .fold(
+                        { handleFailure(params) },
+                        { handleSuccess(params, it.contentUri, uploadedThumbnailUrl) }
+                )
     }
 
-    private fun handleFailure(): Result {
-        return Result.retry()
+    private fun createAttachmentFile(attachment: ContentAttachmentData): File? {
+        return try {
+            File(attachment.path)
+        } catch (e: Exception) {
+            Timber.e(e)
+            null
+        }
     }
 
-    private fun handleSuccess(params: Params, contentUploadResponse: ContentUploadResponse): Result {
-        val event = updateEvent(params.event, contentUploadResponse.contentUri)
+    private fun handleFailure(params: Params): Result {
+        contentUploadProgressTracker.setFailure(params.event.eventId!!)
+        return Result.failure()
+    }
+
+    private fun handleSuccess(params: Params,
+                              attachmentUrl: String,
+                              thumbnailUrl: String?): Result {
+        contentUploadProgressTracker.setFailure(params.event.eventId!!)
+        val event = updateEvent(params.event, attachmentUrl, thumbnailUrl)
         val sendParams = SendEventWorker.Params(params.roomId, event)
         return Result.success(WorkerParamsFactory.toData(sendParams))
     }
 
-    private fun updateEvent(event: Event, url: String): Event {
+    private fun updateEvent(event: Event, url: String, thumbnailUrl: String? = null): Event {
         val messageContent: MessageContent = event.content.toModel() ?: return event
         val updatedContent = when (messageContent) {
             is MessageImageContent -> messageContent.update(url)
-            is MessageVideoContent -> messageContent.update(url)
+            is MessageVideoContent -> messageContent.update(url, thumbnailUrl)
             is MessageFileContent  -> messageContent.update(url)
             is MessageAudioContent -> messageContent.update(url)
             else                   -> messageContent
@@ -84,8 +124,8 @@ internal class UploadContentWorker(context: Context, params: WorkerParameters)
         return copy(url = url)
     }
 
-    private fun MessageVideoContent.update(url: String): MessageVideoContent {
-        return copy(url = url)
+    private fun MessageVideoContent.update(url: String, thumbnailUrl: String?): MessageVideoContent {
+        return copy(url = url, info = info?.copy(thumbnailUrl = thumbnailUrl))
     }
 
     private fun MessageFileContent.update(url: String): MessageFileContent {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt

@@ -16,6 +16,7 @@
 
 package im.vector.matrix.android.internal.session.room.send
 
+import android.media.MediaMetadataRetriever
 import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.api.session.content.ContentAttachmentData
 import im.vector.matrix.android.api.session.events.model.Event
@@ -30,7 +31,9 @@ import im.vector.matrix.android.api.session.room.model.message.MessageImageConte
 import im.vector.matrix.android.api.session.room.model.message.MessageTextContent
 import im.vector.matrix.android.api.session.room.model.message.MessageType
 import im.vector.matrix.android.api.session.room.model.message.MessageVideoContent
+import im.vector.matrix.android.api.session.room.model.message.ThumbnailInfo
 import im.vector.matrix.android.api.session.room.model.message.VideoInfo
+import im.vector.matrix.android.internal.session.content.ThumbnailExtractor
 
 internal class LocalEchoEventFactory(private val credentials: Credentials) {
 
@@ -53,7 +56,7 @@ internal class LocalEchoEventFactory(private val credentials: Credentials) {
                 type = MessageType.MSGTYPE_IMAGE,
                 body = attachment.name ?: "image",
                 info = ImageInfo(
-                        mimeType = attachment.mimeType ?: "image/png",
+                        mimeType = attachment.mimeType,
                         width = attachment.width?.toInt() ?: 0,
                         height = attachment.height?.toInt() ?: 0,
                         size = attachment.size.toInt()
@@ -64,15 +67,35 @@ internal class LocalEchoEventFactory(private val credentials: Credentials) {
     }
 
     private fun createVideoEvent(roomId: String, attachment: ContentAttachmentData): Event {
+        val mediaDataRetriever = MediaMetadataRetriever()
+        mediaDataRetriever.setDataSource(attachment.path)
+
+        // Use frame to calculate height and width as we are sure to get the right ones
+        val firstFrame = mediaDataRetriever.frameAtTime
+        val height = firstFrame.height
+        val width = firstFrame.width
+        mediaDataRetriever.release()
+
+        val thumbnailInfo = ThumbnailExtractor.extractThumbnail(attachment)?.let {
+            ThumbnailInfo(
+                    width = it.width,
+                    height = it.height,
+                    size = it.size,
+                    mimeType = it.mimeType
+            )
+        }
         val content = MessageVideoContent(
                 type = MessageType.MSGTYPE_VIDEO,
                 body = attachment.name ?: "video",
                 info = VideoInfo(
-                        mimeType = attachment.mimeType ?: "video/mpeg",
-                        width = attachment.width?.toInt() ?: 0,
-                        height = attachment.height?.toInt() ?: 0,
+                        mimeType = attachment.mimeType,
+                        width = width,
+                        height = height,
                         size = attachment.size,
-                        duration = attachment.duration?.toInt() ?: 0
+                        duration = attachment.duration?.toInt() ?: 0,
+                        // Glide will be able to use the local path and extract a thumbnail.
+                        thumbnailUrl = attachment.path,
+                        thumbnailInfo = thumbnailInfo
                 ),
                 url = attachment.path
         )