From 30c893da07d976743d11deaf8c114844637752cc Mon Sep 17 00:00:00 2001
From: Taras Smakula <tarassmakula@gmail.com>
Date: Tue, 5 Sep 2023 15:06:01 +0300
Subject: [PATCH] Add image thumbnails
---
.../matrix/android/sdk/api/util/MimeTypes.kt | 1 +
.../session/content/ThumbnailExtractor.kt | 57 +++++++++++++++++--
.../session/content/UploadContentWorker.kt | 15 +++--
.../room/send/LocalEchoEventFactory.kt | 10 ++++
4 files changed, 74 insertions(+), 9 deletions(-)
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MimeTypes.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MimeTypes.kt
index 5ec0deda..af8ab71a 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MimeTypes.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MimeTypes.kt
@@ -30,6 +30,7 @@ object MimeTypes {
const val BadJpg = "image/jpg"
const val Jpeg = "image/jpeg"
const val Gif = "image/gif"
+ const val Webp = "image/webp"
const val Ogg = "audio/ogg"
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/ThumbnailExtractor.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/ThumbnailExtractor.kt
index 55db64f3..de805f59 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/ThumbnailExtractor.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/ThumbnailExtractor.kt
@@ -18,7 +18,12 @@ package org.matrix.android.sdk.internal.session.content
import android.content.Context
import android.graphics.Bitmap
+import android.graphics.ImageDecoder
import android.media.MediaMetadataRetriever
+import android.net.Uri
+import android.os.Build
+import android.provider.MediaStore
+import android.util.Size
import org.matrix.android.sdk.api.session.content.ContentAttachmentData
import org.matrix.android.sdk.api.util.MimeTypes
import timber.log.Timber
@@ -38,10 +43,11 @@ internal class ThumbnailExtractor @Inject constructor(
)
fun extractThumbnail(attachment: ContentAttachmentData): ThumbnailData? {
- return if (attachment.type == ContentAttachmentData.Type.VIDEO) {
- extractVideoThumbnail(attachment)
- } else {
- null
+ if (attachment.mimeType == MimeTypes.Gif || attachment.mimeType == MimeTypes.Webp) return null
+ return when (attachment.type) {
+ ContentAttachmentData.Type.VIDEO -> extractVideoThumbnail(attachment)
+ ContentAttachmentData.Type.IMAGE -> extractImageThumbnail(attachment)
+ else -> null
}
}
@@ -50,7 +56,8 @@ internal class ThumbnailExtractor @Inject constructor(
val mediaMetadataRetriever = MediaMetadataRetriever()
try {
mediaMetadataRetriever.setDataSource(context, attachment.queryUri)
- mediaMetadataRetriever.frameAtTime?.let { thumbnail ->
+ val scaledBitmap = mediaMetadataRetriever.frameAtTime?.let { createScaledThumbnailBitmap(it) }
+ scaledBitmap?.let { thumbnail ->
val outputStream = ByteArrayOutputStream()
thumbnail.compress(Bitmap.CompressFormat.JPEG, 80, outputStream)
val thumbnailWidth = thumbnail.width
@@ -75,4 +82,44 @@ internal class ThumbnailExtractor @Inject constructor(
}
return thumbnailData
}
+
+ private fun extractImageThumbnail(attachment: ContentAttachmentData): ThumbnailData? {
+ var thumbnailData: ThumbnailData? = null
+ try {
+ val thumbnail = createScaledThumbnailBitmap(getBitmapFromUri(attachment.queryUri))
+ val outputStream = ByteArrayOutputStream()
+ thumbnail.compress(Bitmap.CompressFormat.JPEG, 80, outputStream)
+ val thumbnailWidth = thumbnail.width
+ val thumbnailHeight = thumbnail.height
+ val thumbnailSize = outputStream.size()
+ thumbnailData = ThumbnailData(
+ width = thumbnailWidth,
+ height = thumbnailHeight,
+ size = thumbnailSize.toLong(),
+ bytes = outputStream.toByteArray(),
+ mimeType = MimeTypes.Jpeg
+ )
+ thumbnail.recycle()
+ outputStream.reset()
+ } catch (e: Exception) {
+ Timber.e(e, "Cannot extract image thumbnail")
+ }
+ return thumbnailData
+ }
+
+ private fun getBitmapFromUri(uri: Uri) = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+ ImageDecoder.decodeBitmap(ImageDecoder.createSource(context.contentResolver, uri))
+ } else {
+ MediaStore.Images.Media.getBitmap(context.contentResolver, uri)
+ }
+
+ private fun createScaledThumbnailBitmap(originalBitmap: Bitmap): Bitmap {
+ val maxThumbnailSize = 800
+ val originalWidth = originalBitmap.width
+ val originalHeight = originalBitmap.height
+ val aspectRatio = originalWidth.toFloat() / originalHeight.toFloat()
+ val size = if (originalHeight > originalWidth) Size((maxThumbnailSize * aspectRatio).toInt(), maxThumbnailSize)
+ else Size(maxThumbnailSize, (maxThumbnailSize / aspectRatio).toInt())
+ return Bitmap.createScaledBitmap(originalBitmap, size.width, size.height, true)
+ }
}
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/UploadContentWorker.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/UploadContentWorker.kt
index 3dd44073..6aa1fb52 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/UploadContentWorker.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/UploadContentWorker.kt
@@ -185,6 +185,7 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
} else if (attachment.type == ContentAttachmentData.Type.VIDEO &&
// Do not compress gif
attachment.mimeType != MimeTypes.Gif &&
+ attachment.mimeType != MimeTypes.Webp &&
params.compressBeforeSending) {
fileToUpload = videoCompressor.compress(workingFile, object : ProgressListener {
override fun onProgress(progress: Int, total: Int) {
@@ -193,7 +194,7 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
})
.let { videoCompressionResult ->
when (videoCompressionResult) {
- is VideoCompressionResult.Success -> {
+ is VideoCompressionResult.Success -> {
val compressedFile = videoCompressionResult.compressedFile
var compressedWidth: Int? = null
var compressedHeight: Int? = null
@@ -217,10 +218,12 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
compressedFile
.also { filesToDelete.add(it) }
}
+
VideoCompressionResult.CompressionNotNeeded,
VideoCompressionResult.CompressionCancelled -> {
workingFile
}
+
is VideoCompressionResult.CompressionFailed -> {
Timber.e(videoCompressionResult.failure, "Video compression failed")
workingFile
@@ -413,11 +416,11 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
// Retrieve potential additional content from the original event
val additionalContent = content.orEmpty() - messageContent?.toContent().orEmpty().keys
val updatedContent = when (messageContent) {
- is MessageImageContent -> messageContent.update(url, encryptedFileInfo, newAttachmentAttributes)
+ is MessageImageContent -> messageContent.update(url, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo, newAttachmentAttributes)
is MessageVideoContent -> messageContent.update(url, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo, newAttachmentAttributes)
- is MessageFileContent -> messageContent.update(url, encryptedFileInfo, newAttachmentAttributes.newFileSize)
+ is MessageFileContent -> messageContent.update(url, encryptedFileInfo, newAttachmentAttributes.newFileSize)
is MessageAudioContent -> messageContent.update(url, encryptedFileInfo, newAttachmentAttributes.newFileSize)
- else -> messageContent
+ else -> messageContent
}
event.content = ContentMapper.map(updatedContent.toContent().plus(additionalContent))
}
@@ -430,12 +433,16 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
private fun MessageImageContent.update(
url: String,
encryptedFileInfo: EncryptedFileInfo?,
+ thumbnailUrl: String?,
+ thumbnailEncryptedFileInfo: EncryptedFileInfo?,
newAttachmentAttributes: NewAttachmentAttributes?
): MessageImageContent {
return copy(
url = if (encryptedFileInfo == null) url else null,
encryptedFileInfo = encryptedFileInfo?.copy(url = url),
info = info?.copy(
+ thumbnailUrl = if (thumbnailEncryptedFileInfo == null) thumbnailUrl else null,
+ thumbnailFile = thumbnailEncryptedFileInfo?.copy(url = thumbnailUrl),
width = newAttachmentAttributes?.newWidth ?: info.width,
height = newAttachmentAttributes?.newHeight ?: info.height,
size = newAttachmentAttributes?.newFileSize ?: info.size
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoEventFactory.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoEventFactory.kt
index cf0d54d0..05350991 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoEventFactory.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoEventFactory.kt
@@ -427,6 +427,14 @@ internal class LocalEchoEventFactory @Inject constructor(
}
}
+ val thumbnailInfo = thumbnailExtractor.extractThumbnail(attachment)?.let {
+ ThumbnailInfo(
+ width = it.width,
+ height = it.height,
+ size = it.size,
+ mimeType = it.mimeType
+ )
+ }
val content = MessageImageContent(
msgType = MessageType.MSGTYPE_IMAGE,
body = attachment.name ?: "image",
@@ -435,6 +443,8 @@ internal class LocalEchoEventFactory @Inject constructor(
width = width?.toInt() ?: 0,
height = height?.toInt() ?: 0,
size = attachment.size,
+ thumbnailUrl = attachment.queryUri.toString(),
+ thumbnailInfo = thumbnailInfo,
thumbHash = attachment.thumbHash
),
url = attachment.queryUri.toString(),
--
GitLab