Merge branch 'develop' into feature/ons/fix_markdown_formatter

CHANGES.md

@@ -20,9 +20,12 @@ Improvements 🙌:
  - Considerably faster QR-code bitmap generation (#2331)
 
 Bugfix 🐛:
+ - Fixed ringtone handling (#2100 & #2246)
  - Messages encrypted with no way to decrypt after SDK update from 0.18 to 1.0.0 (#2252)
+ - Incoming call continues to ring if call is answered on another device (#1921)
  - Search Result | scroll jumps after pagination (#2238)
  - Badly formatted mentions in body (#1506)
+ - KeysBackup: Avoid using `!!` (#2262)
 
 Translations 🗣:
  -

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/CryptoTestHelper.kt

@@ -245,7 +245,8 @@ class CryptoTestHelper(private val mTestHelper: CommonTestHelper) {
     fun createFakeMegolmBackupCreationInfo(): MegolmBackupCreationInfo {
         return MegolmBackupCreationInfo(
                 algorithm = MXCRYPTO_ALGORITHM_MEGOLM_BACKUP,
-                authData = createFakeMegolmBackupAuthData()
+                authData = createFakeMegolmBackupAuthData(),
+                recoveryKey = "fake"
         )
     }
 

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt

@@ -115,9 +115,8 @@ class KeysBackupTest : InstrumentedTest {
         }
 
         assertEquals(MXCRYPTO_ALGORITHM_MEGOLM_BACKUP, megolmBackupCreationInfo.algorithm)
-        assertNotNull(megolmBackupCreationInfo.authData)
+        assertNotNull(megolmBackupCreationInfo.authData.publicKey)
-        assertNotNull(megolmBackupCreationInfo.authData!!.publicKey)
+        assertNotNull(megolmBackupCreationInfo.authData.signatures)
-        assertNotNull(megolmBackupCreationInfo.authData!!.signatures)
         assertNotNull(megolmBackupCreationInfo.recoveryKey)
 
         stateObserver.stopAndCheckStates(null)
@@ -258,14 +257,14 @@ class KeysBackupTest : InstrumentedTest {
         // - Check encryptGroupSession() returns stg
         val keyBackupData = keysBackup.encryptGroupSession(session)
         assertNotNull(keyBackupData)
-        assertNotNull(keyBackupData.sessionData)
+        assertNotNull(keyBackupData!!.sessionData)
 
         // - Check pkDecryptionFromRecoveryKey() is able to create a OlmPkDecryption
         val decryption = keysBackup.pkDecryptionFromRecoveryKey(keyBackupCreationInfo.recoveryKey)
         assertNotNull(decryption)
         // - Check decryptKeyBackupData() returns stg
         val sessionData = keysBackup
-                .decryptKeyBackupData(keyBackupData,
+                .decryptKeyBackupData(keyBackupData!!,
                         session.olmInboundGroupSession!!.sessionIdentifier(),
                         cryptoTestData.roomId,
                         decryption!!)

matrix-sdk-android/src/debug/java/org/matrix/android/sdk/internal/database/RealmDebugTools.kt

@@ -1,68 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.database
-
-import org.matrix.android.sdk.internal.crypto.store.db.model.CrossSigningInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.DeviceInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.GossipingEventEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.IncomingGossipingRequestEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.KeyInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.KeysBackupDataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OutgoingGossipingRequestEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.TrustLevelEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
-import io.realm.Realm
-import io.realm.RealmConfiguration
-import io.realm.kotlin.where
-import timber.log.Timber
-
-object RealmDebugTools {
-    /**
-     * Log info about the crypto DB
-     */
-    fun dumpCryptoDb(realmConfiguration: RealmConfiguration) {
-        Realm.getInstance(realmConfiguration).use {
-            Timber.d("Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}")
-
-            val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) }
-            Timber.d("Realm encryption key : $key")
-
-            // Check if we have data
-            Timber.e("Realm is empty: ${it.isEmpty}")
-
-            Timber.d("Realm has CryptoMetadataEntity: ${it.where<CryptoMetadataEntity>().count()}")
-            Timber.d("Realm has CryptoRoomEntity: ${it.where<CryptoRoomEntity>().count()}")
-            Timber.d("Realm has DeviceInfoEntity: ${it.where<DeviceInfoEntity>().count()}")
-            Timber.d("Realm has KeysBackupDataEntity: ${it.where<KeysBackupDataEntity>().count()}")
-            Timber.d("Realm has OlmInboundGroupSessionEntity: ${it.where<OlmInboundGroupSessionEntity>().count()}")
-            Timber.d("Realm has OlmSessionEntity: ${it.where<OlmSessionEntity>().count()}")
-            Timber.d("Realm has UserEntity: ${it.where<UserEntity>().count()}")
-            Timber.d("Realm has KeyInfoEntity: ${it.where<KeyInfoEntity>().count()}")
-            Timber.d("Realm has CrossSigningInfoEntity: ${it.where<CrossSigningInfoEntity>().count()}")
-            Timber.d("Realm has TrustLevelEntity: ${it.where<TrustLevelEntity>().count()}")
-            Timber.d("Realm has GossipingEventEntity: ${it.where<GossipingEventEntity>().count()}")
-            Timber.d("Realm has IncomingGossipingRequestEntity: ${it.where<IncomingGossipingRequestEntity>().count()}")
-            Timber.d("Realm has OutgoingGossipingRequestEntity: ${it.where<OutgoingGossipingRequestEntity>().count()}")
-            Timber.d("Realm has MyDeviceLastSeenInfoEntity: ${it.where<MyDeviceLastSeenInfoEntity>().count()}")
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt

@@ -238,4 +238,9 @@ interface Session :
     }
 
     val sharedSecretStorageService: SharedSecretStorageService
+
+    /**
+     * Maintenance API, allows to print outs info on DB size to logcat
+     */
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt

@@ -155,4 +155,6 @@ interface CryptoService {
     // For testing shared session
     fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int>
     fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent?
+
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt

@@ -314,6 +314,10 @@ internal class DefaultCryptoService @Inject constructor(
         }
         // Just update
         fetchDevicesList(NoOpMatrixCallback())
+
+        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            cryptoStore.tidyUpDataBase()
+        }
     }
 
     fun ensureDevice() {
@@ -1291,6 +1295,11 @@ internal class DefaultCryptoService @Inject constructor(
     override fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? {
         return cryptoStore.getWithHeldMegolmSession(roomId, sessionId)
     }
+
+    override fun logDbUsageInfo() {
+        cryptoStore.logDbUsageInfo()
+    }
+
     /* ==========================================================================================
      * For test only
      * ========================================================================================== */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt

@@ -30,7 +30,6 @@ import org.matrix.android.sdk.api.listeners.StepProgressListener
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupService
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupState
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupStateListener
-import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
 import org.matrix.android.sdk.internal.crypto.MegolmSessionData
@@ -85,6 +84,7 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.internal.crypto.keysbackup.model.SignalableMegolmBackupAuthData
 import org.matrix.olm.OlmException
 import org.matrix.olm.OlmPkDecryption
 import org.matrix.olm.OlmPkEncryption
@@ -170,7 +170,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             runCatching {
                 withContext(coroutineDispatchers.crypto) {
                     val olmPkDecryption = OlmPkDecryption()
-                    val megolmBackupAuthData = if (password != null) {
+                    val signalableMegolmBackupAuthData = if (password != null) {
                         // Generate a private key from the password
                         val backgroundProgressListener = if (progressListener == null) {
                             null
@@ -189,7 +189,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                         }
 
                         val generatePrivateKeyResult = generatePrivateKeyWithPassword(password, backgroundProgressListener)
-                        MegolmBackupAuthData(
+                        SignalableMegolmBackupAuthData(
                                 publicKey = olmPkDecryption.setPrivateKey(generatePrivateKeyResult.privateKey),
                                 privateKeySalt = generatePrivateKeyResult.salt,
                                 privateKeyIterations = generatePrivateKeyResult.iterations
@@ -197,14 +197,17 @@ internal class DefaultKeysBackupService @Inject constructor(
                     } else {
                         val publicKey = olmPkDecryption.generateKey()
 
-                        MegolmBackupAuthData(
+                        SignalableMegolmBackupAuthData(
                                 publicKey = publicKey
                         )
                     }
 
-                    val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, megolmBackupAuthData.signalableJSONDictionary())
+                    val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, signalableMegolmBackupAuthData.signalableJSONDictionary())
 
-                    val signedMegolmBackupAuthData = megolmBackupAuthData.copy(
+                    val signedMegolmBackupAuthData = MegolmBackupAuthData(
+                            publicKey = signalableMegolmBackupAuthData.publicKey,
+                            privateKeySalt = signalableMegolmBackupAuthData.privateKeySalt,
+                            privateKeyIterations = signalableMegolmBackupAuthData.privateKeyIterations,
                             signatures = objectSigner.signObject(canonicalJson)
                     )
 
@@ -223,8 +226,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         @Suppress("UNCHECKED_CAST")
         val createKeysBackupVersionBody = CreateKeysBackupVersionBody(
                 algorithm = keysBackupCreationInfo.algorithm,
-                authData = MoshiProvider.providesMoshi().adapter(Map::class.java)
+                authData = keysBackupCreationInfo.authData.toJsonDict()
-                        .fromJson(keysBackupCreationInfo.authData?.toJsonString() ?: "") as JsonDict?
         )
 
         keysBackupStateManager.state = KeysBackupState.Enabling
@@ -245,7 +247,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                                     version = data.version,
                                     // We can consider that the server does not have keys yet
                                     count = 0,
-                                    hash = null
+                                    hash = ""
                             )
 
                             enableKeysBackup(keyBackupVersion)
@@ -267,7 +269,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             withContext(coroutineDispatchers.crypto) {
                 // If we're currently backing up to this backup... stop.
                 // (We start using it automatically in createKeysBackupVersion so this is symmetrical).
-                if (keysBackupVersion != null && version == keysBackupVersion!!.version) {
+                if (keysBackupVersion != null && version == keysBackupVersion?.version) {
                     resetKeysBackupData()
                     keysBackupVersion = null
                     keysBackupStateManager.state = KeysBackupState.Unknown
@@ -408,10 +410,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         val keysBackupVersionTrust = KeysBackupVersionTrust()
         val authData = keysBackupVersion.getAuthDataAsMegolmBackupAuthData()
 
-        if (keysBackupVersion.algorithm == null
+        if (authData == null || authData.publicKey.isEmpty() || authData.signatures.isEmpty()) {
-                || authData == null
-                || authData.publicKey.isEmpty()
-                || authData.signatures.isNullOrEmpty()) {
             Timber.v("getKeysBackupTrust: Key backup is absent or missing required data")
             return keysBackupVersionTrust
         }
@@ -479,7 +478,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             cryptoCoroutineScope.launch(coroutineDispatchers.main) {
                 val updateKeysBackupVersionBody = withContext(coroutineDispatchers.crypto) {
                     // Get current signatures, or create an empty set
-                    val myUserSignatures = authData.signatures?.get(userId)?.toMutableMap() ?: HashMap()
+                    val myUserSignatures = authData.signatures[userId].orEmpty().toMutableMap()
 
                     if (trust) {
                         // Add current device signature
@@ -498,26 +497,23 @@ internal class DefaultKeysBackupService @Inject constructor(
                     // Create an updated version of KeysVersionResult
                     val newMegolmBackupAuthData = authData.copy()
 
-                    val newSignatures = newMegolmBackupAuthData.signatures!!.toMutableMap()
+                    val newSignatures = newMegolmBackupAuthData.signatures.toMutableMap()
                     newSignatures[userId] = myUserSignatures
 
                     val newMegolmBackupAuthDataWithNewSignature = newMegolmBackupAuthData.copy(
                             signatures = newSignatures
                     )
 
-                    val moshi = MoshiProvider.providesMoshi()
-                    val adapter = moshi.adapter(Map::class.java)
-
                     @Suppress("UNCHECKED_CAST")
                     UpdateKeysBackupVersionBody(
                             algorithm = keysBackupVersion.algorithm,
-                            authData = adapter.fromJson(newMegolmBackupAuthDataWithNewSignature.toJsonString()) as Map<String, Any>?,
+                            authData = newMegolmBackupAuthDataWithNewSignature.toJsonDict(),
-                            version = keysBackupVersion.version!!)
+                            version = keysBackupVersion.version)
                 }
 
                 // And send it to the homeserver
                 updateKeysBackupVersionTask
-                        .configureWith(UpdateKeysBackupVersionTask.Params(keysBackupVersion.version!!, updateKeysBackupVersionBody)) {
+                        .configureWith(UpdateKeysBackupVersionTask.Params(keysBackupVersion.version, updateKeysBackupVersionBody)) {
                             this.callback = object : MatrixCallback<Unit> {
                                 override fun onSuccess(data: Unit) {
                                     // Relaunch the state machine on this updated backup version
@@ -688,7 +684,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                 stepProgressListener?.onStepProgress(StepProgressListener.Step.DownloadingKey)
 
                 // Get backed up keys from the homeserver
-                val data = getKeys(sessionId, roomId, keysVersionResult.version!!)
+                val data = getKeys(sessionId, roomId, keysVersionResult.version)
 
                 withContext(coroutineDispatchers.computation) {
                     val sessionsData = ArrayList<MegolmSessionData>()
@@ -1023,19 +1019,10 @@ internal class DefaultKeysBackupService @Inject constructor(
      * @return the authentication if found and valid, null in other case
      */
     private fun getMegolmBackupAuthData(keysBackupData: KeysVersionResult): MegolmBackupAuthData? {
-        if (keysBackupData.version.isNullOrBlank()
+        return keysBackupData
-                || keysBackupData.algorithm != MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
+                .takeIf { it.version.isNotEmpty() && it.algorithm == MXCRYPTO_ALGORITHM_MEGOLM_BACKUP }
-                || keysBackupData.authData == null) {
+                ?.getAuthDataAsMegolmBackupAuthData()
-            return null
+                ?.takeIf { it.publicKey.isNotEmpty() }
-        }
-
-        val authData = keysBackupData.getAuthDataAsMegolmBackupAuthData()
-
-        if (authData?.signatures == null || authData.publicKey.isBlank()) {
-            return null
-        }
-
-        return authData
     }
 
     /**
@@ -1123,34 +1110,29 @@ internal class DefaultKeysBackupService @Inject constructor(
      * @param keysVersionResult backup information object as returned by [getCurrentVersion].
      */
     private fun enableKeysBackup(keysVersionResult: KeysVersionResult) {
-        if (keysVersionResult.authData != null) {
+        val retrievedMegolmBackupAuthData = keysVersionResult.getAuthDataAsMegolmBackupAuthData()
-            val retrievedMegolmBackupAuthData = keysVersionResult.getAuthDataAsMegolmBackupAuthData()
 
-            if (retrievedMegolmBackupAuthData != null) {
+        if (retrievedMegolmBackupAuthData != null) {
-                keysBackupVersion = keysVersionResult
+            keysBackupVersion = keysVersionResult
-                cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-                    cryptoStore.setKeyBackupVersion(keysVersionResult.version)
+                cryptoStore.setKeyBackupVersion(keysVersionResult.version)
-                }
-
-                onServerDataRetrieved(keysVersionResult.count, keysVersionResult.hash)
-
-                try {
-                    backupOlmPkEncryption = OlmPkEncryption().apply {
-                        setRecipientKey(retrievedMegolmBackupAuthData.publicKey)
-                    }
-                } catch (e: OlmException) {
-                    Timber.e(e, "OlmException")
-                    keysBackupStateManager.state = KeysBackupState.Disabled
-                    return
-                }
-
-                keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
-
-                maybeBackupKeys()
-            } else {
-                Timber.e("Invalid authentication data")
-                keysBackupStateManager.state = KeysBackupState.Disabled
             }
+
+            onServerDataRetrieved(keysVersionResult.count, keysVersionResult.hash)
+
+            try {
+                backupOlmPkEncryption = OlmPkEncryption().apply {
+                    setRecipientKey(retrievedMegolmBackupAuthData.publicKey)
+                }
+            } catch (e: OlmException) {
+                Timber.e(e, "OlmException")
+                keysBackupStateManager.state = KeysBackupState.Disabled
+                return
+            }
+
+            keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
+
+            maybeBackupKeys()
         } else {
             Timber.e("Invalid authentication data")
             keysBackupStateManager.state = KeysBackupState.Disabled
@@ -1160,11 +1142,11 @@ internal class DefaultKeysBackupService @Inject constructor(
     /**
      * Update the DB with data fetch from the server
      */
-    private fun onServerDataRetrieved(count: Int?, hash: String?) {
+    private fun onServerDataRetrieved(count: Int?, etag: String?) {
         cryptoStore.setKeysBackupData(KeysBackupDataEntity()
                 .apply {
                     backupLastServerNumberOfKeys = count
-                    backupLastServerHash = hash
+                    backupLastServerHash = etag
                 }
         )
     }
@@ -1179,6 +1161,7 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         cryptoStore.setKeyBackupVersion(null)
         cryptoStore.setKeysBackupData(null)
+        backupOlmPkEncryption?.releaseEncryption()
         backupOlmPkEncryption = null
 
         // Reset backup markers
@@ -1229,22 +1212,19 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                 // Gather data to send to the homeserver
                 // roomId -> sessionId -> MXKeyBackupData
-                val keysBackupData = KeysBackupData(
+                val keysBackupData = KeysBackupData()
-                        roomIdToRoomKeysBackupData = HashMap()
-                )
 
-                for (olmInboundGroupSessionWrapper in olmInboundGroupSessionWrappers) {
+                olmInboundGroupSessionWrappers.forEach { olmInboundGroupSessionWrapper ->
-                    val keyBackupData = encryptGroupSession(olmInboundGroupSessionWrapper)
+                    val roomId = olmInboundGroupSessionWrapper.roomId ?: return@forEach
-                    if (keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId] == null) {
+                    val olmInboundGroupSession = olmInboundGroupSessionWrapper.olmInboundGroupSession ?: return@forEach
-                        val roomKeysBackupData = RoomKeysBackupData(
-                                sessionIdToKeyBackupData = HashMap()
-                        )
-                        keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId!!] = roomKeysBackupData
-                    }
 
                     try {
-                        keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId]!!
+                        encryptGroupSession(olmInboundGroupSessionWrapper)
-                                .sessionIdToKeyBackupData[olmInboundGroupSessionWrapper.olmInboundGroupSession!!.sessionIdentifier()] = keyBackupData
+                                ?.let {
+                                    keysBackupData.roomIdToRoomKeysBackupData
+                                            .getOrPut(roomId) { RoomKeysBackupData() }
+                                            .sessionIdToKeyBackupData[olmInboundGroupSession.sessionIdentifier()] = it
+                                }
                     } catch (e: OlmException) {
                         Timber.e(e, "OlmException")
                     }
@@ -1252,71 +1232,71 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                 Timber.v("backupKeys: 4 - Sending request")
 
-                val sendingRequestCallback = object : MatrixCallback<BackupKeysResult> {
+                // Make the request
-                    override fun onSuccess(data: BackupKeysResult) {
+                val version = keysBackupVersion?.version ?: return@withContext
-                        uiHandler.post {
-                            Timber.v("backupKeys: 5a - Request complete")
 
-                            // Mark keys as backed up
+                storeSessionDataTask
-                            cryptoStore.markBackupDoneForInboundGroupSessions(olmInboundGroupSessionWrappers)
+                        .configureWith(StoreSessionsDataTask.Params(version, keysBackupData)) {
+                            this.callback = object : MatrixCallback<BackupKeysResult> {
+                                override fun onSuccess(data: BackupKeysResult) {
+                                    uiHandler.post {
+                                        Timber.v("backupKeys: 5a - Request complete")
 
-                            if (olmInboundGroupSessionWrappers.size < KEY_BACKUP_SEND_KEYS_MAX_COUNT) {
+                                        // Mark keys as backed up
-                                Timber.v("backupKeys: All keys have been backed up")
+                                        cryptoStore.markBackupDoneForInboundGroupSessions(olmInboundGroupSessionWrappers)
-                                onServerDataRetrieved(data.count, data.hash)
 
-                                // Note: Changing state will trigger the call to backupAllGroupSessionsCallback.onSuccess()
+                                        if (olmInboundGroupSessionWrappers.size < KEY_BACKUP_SEND_KEYS_MAX_COUNT) {
-                                keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
+                                            Timber.v("backupKeys: All keys have been backed up")
-                            } else {
+                                            onServerDataRetrieved(data.count, data.hash)
-                                Timber.v("backupKeys: Continue to back up keys")
-                                keysBackupStateManager.state = KeysBackupState.WillBackUp
 
-                                backupKeys()
+                                            // Note: Changing state will trigger the call to backupAllGroupSessionsCallback.onSuccess()
-                            }
+                                            keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
-                        }
+                                        } else {
-                    }
+                                            Timber.v("backupKeys: Continue to back up keys")
+                                            keysBackupStateManager.state = KeysBackupState.WillBackUp
 
-                    override fun onFailure(failure: Throwable) {
+                                            backupKeys()
-                        if (failure is Failure.ServerError) {
+                                        }
-                            uiHandler.post {
+                                    }
-                                Timber.e(failure, "backupKeys: backupKeys failed.")
+                                }
-
+
-                                when (failure.error.code) {
+                                override fun onFailure(failure: Throwable) {
-                                    MatrixError.M_NOT_FOUND,
+                                    if (failure is Failure.ServerError) {
-                                    MatrixError.M_WRONG_ROOM_KEYS_VERSION -> {
+                                        uiHandler.post {
-                                        // Backup has been deleted on the server, or we are not using the last backup version
+                                            Timber.e(failure, "backupKeys: backupKeys failed.")
-                                        keysBackupStateManager.state = KeysBackupState.WrongBackUpVersion
+
-                                        backupAllGroupSessionsCallback?.onFailure(failure)
+                                            when (failure.error.code) {
-                                        resetBackupAllGroupSessionsListeners()
+                                                MatrixError.M_NOT_FOUND,
-                                        resetKeysBackupData()
+                                                MatrixError.M_WRONG_ROOM_KEYS_VERSION -> {
-                                        keysBackupVersion = null
+                                                    // Backup has been deleted on the server, or we are not using the last backup version
-
+                                                    keysBackupStateManager.state = KeysBackupState.WrongBackUpVersion
-                                        // Do not stay in KeysBackupState.WrongBackUpVersion but check what is available on the homeserver
+                                                    backupAllGroupSessionsCallback?.onFailure(failure)
-                                        checkAndStartKeysBackup()
+                                                    resetBackupAllGroupSessionsListeners()
+                                                    resetKeysBackupData()
+                                                    keysBackupVersion = null
+
+                                                    // Do not stay in KeysBackupState.WrongBackUpVersion but check what is available on the homeserver
+                                                    checkAndStartKeysBackup()
+                                                }
+                                                else                                  ->
+                                                    // Come back to the ready state so that we will retry on the next received key
+                                                    keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
+                                            }
+                                        }
+                                    } else {
+                                        uiHandler.post {
+                                            backupAllGroupSessionsCallback?.onFailure(failure)
+                                            resetBackupAllGroupSessionsListeners()
+
+                                            Timber.e("backupKeys: backupKeys failed.")
+
+                                            // Retry a bit later
+                                            keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
+                                            maybeBackupKeys()
+                                        }
                                     }
-                                    else                                  ->
-                                        // Come back to the ready state so that we will retry on the next received key
-                                        keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
                                 }
                             }
-                        } else {
-                            uiHandler.post {
-                                backupAllGroupSessionsCallback?.onFailure(failure)
-                                resetBackupAllGroupSessionsListeners()
-
-                                Timber.e("backupKeys: backupKeys failed.")
-
-                                // Retry a bit later
-                                keysBackupStateManager.state = KeysBackupState.ReadyToBackUp
-                                maybeBackupKeys()
-                            }
-                        }
-                    }
-                }
-
-                // Make the request
-                storeSessionDataTask
-                        .configureWith(StoreSessionsDataTask.Params(keysBackupVersion!!.version!!, keysBackupData)) {
-                            this.callback = sendingRequestCallback
                         }
                         .executeBy(taskExecutor)
             }
@@ -1325,47 +1305,45 @@ internal class DefaultKeysBackupService @Inject constructor(
 
     @VisibleForTesting
     @WorkerThread
-    fun encryptGroupSession(olmInboundGroupSessionWrapper: OlmInboundGroupSessionWrapper2): KeyBackupData {
+    fun encryptGroupSession(olmInboundGroupSessionWrapper: OlmInboundGroupSessionWrapper2): KeyBackupData? {
         // Gather information for each key
-        val device = cryptoStore.deviceWithIdentityKey(olmInboundGroupSessionWrapper.senderKey!!)
+        val device = olmInboundGroupSessionWrapper.senderKey?.let { cryptoStore.deviceWithIdentityKey(it) }
 
         // Build the m.megolm_backup.v1.curve25519-aes-sha2 data as defined at
         // https://github.com/uhoreg/matrix-doc/blob/e2e_backup/proposals/1219-storing-megolm-keys-serverside.md#mmegolm_backupv1curve25519-aes-sha2-key-format
-        val sessionData = olmInboundGroupSessionWrapper.exportKeys()
+        val sessionData = olmInboundGroupSessionWrapper.exportKeys() ?: return null
         val sessionBackupData = mapOf(
-                "algorithm" to sessionData!!.algorithm,
+                "algorithm" to sessionData.algorithm,
                 "sender_key" to sessionData.senderKey,
                 "sender_claimed_keys" to sessionData.senderClaimedKeys,
-                "forwarding_curve25519_key_chain" to (sessionData.forwardingCurve25519KeyChain
+                "forwarding_curve25519_key_chain" to (sessionData.forwardingCurve25519KeyChain.orEmpty()),
-                        ?: ArrayList<Any>()),
                 "session_key" to sessionData.sessionKey)
 
-        var encryptedSessionBackupData: OlmPkMessage? = null
+        val json = MoshiProvider.providesMoshi()
+                .adapter(Map::class.java)
+                .toJson(sessionBackupData)
 
-        val moshi = MoshiProvider.providesMoshi()
+        val encryptedSessionBackupData = try {
-        val adapter = moshi.adapter(Map::class.java)
+            backupOlmPkEncryption?.encrypt(json)
-
-        try {
-            val json = adapter.toJson(sessionBackupData)
-
-            encryptedSessionBackupData = backupOlmPkEncryption?.encrypt(json)
         } catch (e: OlmException) {
             Timber.e(e, "OlmException")
+            null
         }
+                ?: return null
 
         // Build backup data for that key
         return KeyBackupData(
                 firstMessageIndex = try {
-                    olmInboundGroupSessionWrapper.olmInboundGroupSession!!.firstKnownIndex
+                    olmInboundGroupSessionWrapper.olmInboundGroupSession?.firstKnownIndex ?: 0
                 } catch (e: OlmException) {
                     Timber.e(e, "OlmException")
                     0L
                 },
-                forwardedCount = olmInboundGroupSessionWrapper.forwardingCurve25519KeyChain!!.size,
+                forwardedCount = olmInboundGroupSessionWrapper.forwardingCurve25519KeyChain.orEmpty().size,
                 isVerified = device?.isVerified == true,
 
                 sessionData = mapOf(
-                        "ciphertext" to encryptedSessionBackupData!!.mCipherText,
+                        "ciphertext" to encryptedSessionBackupData.mCipherText,
                         "mac" to encryptedSessionBackupData.mMac,
                         "ephemeral" to encryptedSessionBackupData.mEphemeralKey)
         )
@@ -1378,9 +1356,9 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         val jsonObject = keyBackupData.sessionData
 
-        val ciphertext = jsonObject?.get("ciphertext")?.toString()
+        val ciphertext = jsonObject["ciphertext"]?.toString()
-        val mac = jsonObject?.get("mac")?.toString()
+        val mac = jsonObject["mac"]?.toString()
-        val ephemeralKey = jsonObject?.get("ephemeral")?.toString()
+        val ephemeralKey = jsonObject["ephemeral"]?.toString()
 
         if (ciphertext != null && mac != null && ephemeralKey != null) {
             val encrypted = OlmPkMessage()
@@ -1425,8 +1403,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         @Suppress("UNCHECKED_CAST")
         val createKeysBackupVersionBody = CreateKeysBackupVersionBody(
                 algorithm = keysBackupCreationInfo.algorithm,
-                authData = MoshiProvider.providesMoshi().adapter(Map::class.java)
+                authData = keysBackupCreationInfo.authData.toJsonDict()
-                        .fromJson(keysBackupCreationInfo.authData?.toJsonString() ?: "") as JsonDict?
         )
 
         createKeysBackupVersionTask

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/api/RoomKeysApi.kt

@@ -35,7 +35,7 @@ import retrofit2.http.Path
 import retrofit2.http.Query
 
 /**
- * Ref: https://github.com/uhoreg/matrix-doc/blob/e2e_backup/proposals/1219-storing-megolm-keys-serverside.md
+ * Ref: https://matrix.org/docs/spec/client_server/unstable#server-side-key-backups
  */
 internal interface RoomKeysApi {
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/MegolmBackupAuthData.kt

@@ -18,6 +18,7 @@ package org.matrix.android.sdk.internal.crypto.keysbackup.model
 
 import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.di.MoshiProvider
 
 /**
@@ -30,7 +31,7 @@ data class MegolmBackupAuthData(
          * The curve25519 public key used to encrypt the backups.
          */
         @Json(name = "public_key")
-        val publicKey: String = "",
+        val publicKey: String,
 
         /**
          * In case of a backup created from a password, the salt associated with the backup
@@ -50,20 +51,38 @@ data class MegolmBackupAuthData(
          * userId -> (deviceSignKeyId -> signature)
          */
         @Json(name = "signatures")
-        val signatures: Map<String, Map<String, String>>? = null
+        val signatures: Map<String, Map<String, String>>
 ) {
 
-    fun toJsonString(): String {
+    fun toJsonDict(): JsonDict {
-        return MoshiProvider.providesMoshi()
+        val moshi = MoshiProvider.providesMoshi()
+        val adapter = moshi.adapter(Map::class.java)
+
+        return moshi
                 .adapter(MegolmBackupAuthData::class.java)
                 .toJson(this)
+                .let {
+                    @Suppress("UNCHECKED_CAST")
+                    adapter.fromJson(it) as JsonDict
+                }
     }
 
-    /**
+    fun signalableJSONDictionary(): JsonDict {
-     * Same as the parent [MXJSONModel JSONDictionary] but return only
+        return SignalableMegolmBackupAuthData(
-     * data that must be signed.
+                publicKey = publicKey,
-     */
+                privateKeySalt = privateKeySalt,
-    fun signalableJSONDictionary(): Map<String, Any> = HashMap<String, Any>().apply {
+                privateKeyIterations = privateKeyIterations
+        )
+                .signalableJSONDictionary()
+    }
+}
+
+internal data class SignalableMegolmBackupAuthData(
+        val publicKey: String,
+        val privateKeySalt: String? = null,
+        val privateKeyIterations: Int? = null
+) {
+    fun signalableJSONDictionary(): JsonDict = HashMap<String, Any>().apply {
         put("public_key", publicKey)
 
         privateKeySalt?.let {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/MegolmBackupCreationInfo.kt

@@ -23,15 +23,15 @@ data class MegolmBackupCreationInfo(
         /**
          * The algorithm used for storing backups [org.matrix.androidsdk.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP].
          */
-        val algorithm: String = "",
+        val algorithm: String,
 
         /**
          * Authentication data.
          */
-        val authData: MegolmBackupAuthData? = null,
+        val authData: MegolmBackupAuthData,
 
         /**
          * The Base58 recovery key.
          */
-        val recoveryKey: String = ""
+        val recoveryKey: String
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/BackupKeysResult.kt

@@ -16,15 +16,16 @@
 
 package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
+import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
 
 @JsonClass(generateAdapter = true)
-data class BackupKeysResult(
+internal data class BackupKeysResult(
-
         // The hash value which is an opaque string representing stored keys in the backup
-        var hash: String? = null,
+        @Json(name = "etag")
+        val hash: String,
 
         // The number of keys stored in the backup.
-        var count: Int? = null
+        @Json(name = "count")
-
+        val count: Int
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/CreateKeysBackupVersionBody.kt

@@ -21,17 +21,17 @@ import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.util.JsonDict
 
 @JsonClass(generateAdapter = true)
-data class CreateKeysBackupVersionBody(
+internal data class CreateKeysBackupVersionBody(
         /**
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null
+        override val authData: JsonDict
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeyBackupData.kt

@@ -18,7 +18,7 @@ package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
 import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.internal.di.MoshiProvider
+import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.network.parsing.ForceToBoolean
 
 /**
@@ -30,13 +30,13 @@ data class KeyBackupData(
          * Required. The index of the first message in the session that the key can decrypt.
          */
         @Json(name = "first_message_index")
-        val firstMessageIndex: Long = 0,
+        val firstMessageIndex: Long,
 
         /**
          * Required. The number of times this key has been forwarded.
          */
         @Json(name = "forwarded_count")
-        val forwardedCount: Int = 0,
+        val forwardedCount: Int,
 
         /**
          * Whether the device backing up the key has verified the device that the key is from.
@@ -44,16 +44,11 @@ data class KeyBackupData(
          */
         @ForceToBoolean
         @Json(name = "is_verified")
-        val isVerified: Boolean = false,
+        val isVerified: Boolean,
 
         /**
          * Algorithm-dependent data.
          */
         @Json(name = "session_data")
-        val sessionData: Map<String, Any>? = null
+        val sessionData: JsonDict
-) {
+)
-
-    fun toJsonString(): String {
-        return MoshiProvider.providesMoshi().adapter(KeyBackupData::class.java).toJson(this)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysAlgorithmAndData.kt

@@ -17,6 +17,7 @@
 package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
 import org.matrix.android.sdk.api.util.JsonDict
+import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.MegolmBackupAuthData
 import org.matrix.android.sdk.internal.di.MoshiProvider
 
@@ -37,24 +38,25 @@ import org.matrix.android.sdk.internal.di.MoshiProvider
  *     }
  * </pre>
  */
-interface KeysAlgorithmAndData {
+internal interface KeysAlgorithmAndData {
 
     /**
      * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
      */
-    val algorithm: String?
+    val algorithm: String
 
     /**
      * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2" see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
      */
-    val authData: JsonDict?
+    val authData: JsonDict
 
     /**
      * Facility method to convert authData to a MegolmBackupAuthData object
      */
     fun getAuthDataAsMegolmBackupAuthData(): MegolmBackupAuthData? {
         return MoshiProvider.providesMoshi()
-                .adapter(MegolmBackupAuthData::class.java)
+                .takeIf { algorithm == MXCRYPTO_ALGORITHM_MEGOLM_BACKUP }
-                .fromJsonValue(authData)
+                ?.adapter(MegolmBackupAuthData::class.java)
+                ?.fromJsonValue(authData)
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysVersion.kt

@@ -23,5 +23,5 @@ import com.squareup.moshi.JsonClass
 data class KeysVersion(
         // the keys backup version
         @Json(name = "version")
-        val version: String? = null
+        val version: String
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysVersionResult.kt

@@ -26,24 +26,24 @@ data class KeysVersionResult(
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null,
+        override val authData: JsonDict,
 
         // the backup version
         @Json(name = "version")
-        val version: String? = null,
+        val version: String,
 
         // The hash value which is an opaque string representing stored keys in the backup
-        @Json(name = "hash")
+        @Json(name = "etag")
-        val hash: String? = null,
+        val hash: String,
 
         // The number of keys stored in the backup.
         @Json(name = "count")
-        val count: Int? = null
+        val count: Int
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/UpdateKeysBackupVersionBody.kt

@@ -26,16 +26,16 @@ data class UpdateKeysBackupVersionBody(
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null,
+        override val authData: JsonDict,
 
-        // the backup version, mandatory
+        // Optional. The backup version. If present, must be the same as the path parameter.
         @Json(name = "version")
-        val version: String
+        val version: String? = null
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/OlmInboundGroupSessionWrapper2.kt

@@ -48,15 +48,12 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
      */
     val firstKnownIndex: Long?
         get() {
-            if (null != olmInboundGroupSession) {
+            return try {
-                try {
+                olmInboundGroupSession?.firstKnownIndex
-                    return olmInboundGroupSession!!.firstKnownIndex
+            } catch (e: Exception) {
-                } catch (e: Exception) {
+                Timber.e(e, "## getFirstKnownIndex() : getFirstKnownIndex failed")
-                    Timber.e(e, "## getFirstKnownIndex() : getFirstKnownIndex failed")
+                null
-                }
             }
-
-            return null
         }
 
     /**
@@ -90,11 +87,13 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
     @Throws(Exception::class)
     constructor(megolmSessionData: MegolmSessionData) {
         try {
-            olmInboundGroupSession = OlmInboundGroupSession.importSession(megolmSessionData.sessionKey!!)
+            val safeSessionKey = megolmSessionData.sessionKey ?: throw Exception("invalid data")
-
+            olmInboundGroupSession = OlmInboundGroupSession.importSession(safeSessionKey)
-            if (olmInboundGroupSession!!.sessionIdentifier() != megolmSessionData.sessionId) {
+                    .also {
-                throw Exception("Mismatched group session Id")
+                        if (it.sessionIdentifier() != megolmSessionData.sessionId) {
-            }
+                            throw Exception("Mismatched group session Id")
+                        }
+                    }
 
             senderKey = megolmSessionData.senderKey
             keysClaimed = megolmSessionData.senderClaimedKeys
@@ -120,16 +119,18 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
                 return null
             }
 
-            val wantedIndex = index ?: olmInboundGroupSession!!.firstKnownIndex
+            val safeOlmInboundGroupSession = olmInboundGroupSession ?: return null
+
+            val wantedIndex = index ?: safeOlmInboundGroupSession.firstKnownIndex
 
             MegolmSessionData(
                     senderClaimedEd25519Key = keysClaimed?.get("ed25519"),
-                    forwardingCurve25519KeyChain = ArrayList(forwardingCurve25519KeyChain!!),
+                    forwardingCurve25519KeyChain = forwardingCurve25519KeyChain?.toList().orEmpty(),
                     senderKey = senderKey,
                     senderClaimedKeys = keysClaimed,
                     roomId = roomId,
-                    sessionId = olmInboundGroupSession!!.sessionIdentifier(),
+                    sessionId = safeOlmInboundGroupSession.sessionIdentifier(),
-                    sessionKey = olmInboundGroupSession!!.export(wantedIndex),
+                    sessionKey = safeOlmInboundGroupSession.export(wantedIndex),
                     algorithm = MXCRYPTO_ALGORITHM_MEGOLM
             )
         } catch (e: Exception) {
@@ -145,14 +146,11 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
      * @return the exported data
      */
     fun exportSession(messageIndex: Long): String? {
-        if (null != olmInboundGroupSession) {
+        return try {
-            try {
+            return olmInboundGroupSession?.export(messageIndex)
-                return olmInboundGroupSession!!.export(messageIndex)
+        } catch (e: Exception) {
-            } catch (e: Exception) {
+            Timber.e(e, "## exportSession() : export failed")
-                Timber.e(e, "## exportSession() : export failed")
+            null
-            }
         }
-
-        return null
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt

@@ -456,4 +456,6 @@ internal interface IMXCryptoStore {
 
     fun setDeviceKeysUploaded(uploaded: Boolean)
     fun getDeviceKeysUploaded(): Boolean
+    fun tidyUpDataBase()
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt

@@ -87,6 +87,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.query.get
 import org.matrix.android.sdk.internal.crypto.store.db.query.getById
 import org.matrix.android.sdk.internal.crypto.store.db.query.getOrCreate
 import org.matrix.android.sdk.internal.database.mapper.ContentMapper
+import org.matrix.android.sdk.internal.database.tools.RealmDebugTools
 import org.matrix.android.sdk.internal.di.CryptoDatabase
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.MoshiProvider
@@ -1666,4 +1667,48 @@ internal class RealmCryptoStore @Inject constructor(
             result
         }
     }
+
+    /**
+     * Some entries in the DB can get a bit out of control with time
+     * So we need to tidy up a bit
+     */
+    override fun tidyUpDataBase() {
+        val prevWeekTs = System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1_000
+        doRealmTransaction(realmConfiguration) { realm ->
+
+            // Only keep one week history
+            realm.where<IncomingGossipingRequestEntity>()
+                    .lessThan(IncomingGossipingRequestEntityFields.LOCAL_CREATION_TIMESTAMP, prevWeekTs)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} IncomingGossipingRequestEntity")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Clean the cancelled ones?
+            realm.where<OutgoingGossipingRequestEntity>()
+                    .equalTo(OutgoingGossipingRequestEntityFields.REQUEST_STATE_STR, OutgoingGossipingRequestState.CANCELLED.name)
+                    .equalTo(OutgoingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} OutgoingGossipingRequestEntity")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Only keep one week history
+            realm.where<GossipingEventEntity>()
+                    .lessThan(GossipingEventEntityFields.AGE_LOCAL_TS, prevWeekTs)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} GossipingEventEntityFields")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Can we do something for WithHeldSessionEntity?
+        }
+    }
+
+    /**
+     * Prints out database info
+     */
+    override fun logDbUsageInfo() {
+        RealmDebugTools(realmConfiguration).logInfo("Crypto")
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/tools/RealmDebugTools.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database.tools
+
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import org.matrix.android.sdk.BuildConfig
+import timber.log.Timber
+
+internal class RealmDebugTools(
+        private val realmConfiguration: RealmConfiguration
+) {
+    /**
+     * Log info about the DB
+     */
+    fun logInfo(baseName: String) {
+        buildString {
+            append("\n$baseName Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}")
+
+            if (BuildConfig.LOG_PRIVATE_DATA) {
+                val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) }
+                append("\n$baseName Realm encryption key : $key")
+            }
+
+            Realm.getInstance(realmConfiguration).use { realm ->
+                // Check if we have data
+                separator()
+                separator()
+                append("\n$baseName Realm is empty: ${realm.isEmpty}")
+                var total = 0L
+                val maxNameLength = realmConfiguration.realmObjectClasses.maxOf { it.simpleName.length }
+                realmConfiguration.realmObjectClasses.forEach { modelClazz ->
+                    val count = realm.where(modelClazz).count()
+                    total += count
+                    append("\n$baseName Realm - count ${modelClazz.simpleName.padEnd(maxNameLength)} : $count")
+                }
+                separator()
+                append("\n$baseName Realm - total count: $total")
+                separator()
+                separator()
+            }
+        }
+                .let { Timber.i(it) }
+    }
+
+    private fun StringBuilder.separator() = append("\n==============================================")
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt

@@ -59,6 +59,7 @@ import org.matrix.android.sdk.api.session.user.UserService
 import org.matrix.android.sdk.api.session.widgets.WidgetService
 import org.matrix.android.sdk.internal.auth.SessionParamsStore
 import org.matrix.android.sdk.internal.crypto.DefaultCryptoService
+import org.matrix.android.sdk.internal.database.tools.RealmDebugTools
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.SessionId
 import org.matrix.android.sdk.internal.di.UnauthenticatedWithCertificate
@@ -197,7 +198,7 @@ internal class DefaultSession @Inject constructor(
     override fun close() {
         assert(isOpen)
         stopSync()
-       // timelineEventDecryptor.destroy()
+        // timelineEventDecryptor.destroy()
         uiHandler.post {
             lifecycleObservers.forEach { it.onStop() }
         }
@@ -284,4 +285,8 @@ internal class DefaultSession @Inject constructor(
     override fun toString(): String {
         return "$myUserId - ${sessionParams.deviceId}"
     }
+
+    override fun logDbUsageInfo() {
+        RealmDebugTools(realmConfiguration).logInfo("Session")
+    }
 }

vector/src/main/java/im/vector/app/VectorApplication.kt

@@ -42,6 +42,7 @@ import im.vector.app.core.di.HasVectorInjector
 import im.vector.app.core.di.VectorComponent
 import im.vector.app.core.extensions.configureAndStart
 import im.vector.app.core.rx.RxConfig
+import im.vector.app.features.call.WebRtcPeerConnectionManager
 import im.vector.app.features.configuration.VectorConfiguration
 import im.vector.app.features.disclaimer.doNotShowDisclaimerDialog
 import im.vector.app.features.lifecycle.VectorActivityLifecycleCallbacks
@@ -89,6 +90,7 @@ class VectorApplication :
     @Inject lateinit var rxConfig: RxConfig
     @Inject lateinit var popupAlertManager: PopupAlertManager
     @Inject lateinit var pinLocker: PinLocker
+    @Inject lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager
 
     lateinit var vectorComponent: VectorComponent
 
@@ -173,6 +175,7 @@ class VectorApplication :
         })
         ProcessLifecycleOwner.get().lifecycle.addObserver(appStateHandler)
         ProcessLifecycleOwner.get().lifecycle.addObserver(pinLocker)
+        ProcessLifecycleOwner.get().lifecycle.addObserver(webRtcPeerConnectionManager)
         // This should be done as early as possible
         // initKnownEmojiHashSet(appContext)
 

vector/src/main/java/im/vector/app/core/services/CallRingPlayer.kt

@@ -17,6 +17,8 @@
 package im.vector.app.core.services
 
 import android.content.Context
+import android.media.Ringtone
+import android.media.RingtoneManager
 import android.media.AudioAttributes
 import android.media.AudioManager
 import android.media.MediaPlayer
@@ -25,7 +27,26 @@ import androidx.core.content.getSystemService
 import im.vector.app.R
 import timber.log.Timber
 
-class CallRingPlayer(
+class CallRingPlayerIncoming(
+        context: Context
+) {
+
+    private val applicationContext = context.applicationContext
+    private var r: Ringtone? = null
+
+    fun start() {
+        val notification = RingtoneManager.getDefaultUri(RingtoneManager.TYPE_RINGTONE)
+        r = RingtoneManager.getRingtone(applicationContext, notification)
+        Timber.v("## VOIP Starting ringing incomming")
+        r?.play()
+    }
+
+    fun stop() {
+        r?.stop()
+    }
+}
+
+class CallRingPlayerOutgoing(
         context: Context
 ) {
 
@@ -44,12 +65,12 @@ class CallRingPlayer(
             try {
                 if (player?.isPlaying == false) {
                     player?.start()
-                    Timber.v("## VOIP Starting ringing")
+                    Timber.v("## VOIP Starting ringing outgoing")
                 } else {
                     Timber.v("## VOIP already playing")
                 }
             } catch (failure: Throwable) {
-                Timber.e(failure, "## VOIP Failed to start ringing")
+                Timber.e(failure, "## VOIP Failed to start ringing outgoing")
                 player = null
             }
         } else {
@@ -74,7 +95,7 @@ class CallRingPlayer(
             } else {
                 mediaPlayer.setAudioAttributes(AudioAttributes.Builder()
                         .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
-                        .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION_SIGNALLING)
+                        .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
                         .build())
             }
             return mediaPlayer

vector/src/main/java/im/vector/app/core/services/CallService.kt

@@ -40,7 +40,8 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
     private lateinit var notificationUtils: NotificationUtils
     private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager
 
-    private var callRingPlayer: CallRingPlayer? = null
+    private var callRingPlayerIncoming: CallRingPlayerIncoming? = null
+    private var callRingPlayerOutgoing: CallRingPlayerOutgoing? = null
 
     private var wiredHeadsetStateReceiver: WiredHeadsetStateReceiver? = null
     private var bluetoothHeadsetStateReceiver: BluetoothHeadsetReceiver? = null
@@ -63,14 +64,16 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
         super.onCreate()
         notificationUtils = vectorComponent().notificationUtils()
         webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager()
-        callRingPlayer = CallRingPlayer(applicationContext)
+        callRingPlayerIncoming = CallRingPlayerIncoming(applicationContext)
+        callRingPlayerOutgoing = CallRingPlayerOutgoing(applicationContext)
         wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this)
         bluetoothHeadsetStateReceiver = BluetoothHeadsetReceiver.createAndRegister(this, this)
     }
 
     override fun onDestroy() {
         super.onDestroy()
-        callRingPlayer?.stop()
+        callRingPlayerIncoming?.stop()
+        callRingPlayerOutgoing?.stop()
         wiredHeadsetStateReceiver?.let { WiredHeadsetStateReceiver.unRegister(this, it) }
         wiredHeadsetStateReceiver = null
         bluetoothHeadsetStateReceiver?.let { BluetoothHeadsetReceiver.unRegister(this, it) }
@@ -100,16 +103,17 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
         when (intent.action) {
             ACTION_INCOMING_RINGING_CALL -> {
                 mediaSession?.isActive = true
-                callRingPlayer?.start()
+                callRingPlayerIncoming?.start()
                 displayIncomingCallNotification(intent)
             }
             ACTION_OUTGOING_RINGING_CALL -> {
                 mediaSession?.isActive = true
-                callRingPlayer?.start()
+                callRingPlayerOutgoing?.start()
                 displayOutgoingRingingCallNotification(intent)
             }
             ACTION_ONGOING_CALL          -> {
-                callRingPlayer?.stop()
+                callRingPlayerIncoming?.stop()
+                callRingPlayerOutgoing?.stop()
                 displayCallInProgressNotification(intent)
             }
             ACTION_NO_ACTIVE_CALL        -> hideCallNotifications()
@@ -117,7 +121,8 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
                 // lower notification priority
                 displayCallInProgressNotification(intent)
                 // stop ringing
-                callRingPlayer?.stop()
+                callRingPlayerIncoming?.stop()
+                callRingPlayerOutgoing?.stop()
             }
             ACTION_ONGOING_CALL_BG       -> {
                 // there is an ongoing call but call activity is in background
@@ -125,7 +130,8 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
             }
             else                         -> {
                 // Should not happen
-                callRingPlayer?.stop()
+                callRingPlayerIncoming?.stop()
+                callRingPlayerOutgoing?.stop()
                 myStopSelf()
             }
         }

vector/src/main/java/im/vector/app/features/call/CallAudioManager.kt

@@ -93,6 +93,10 @@ class CallAudioManager(
 
     fun startForCall(mxCall: MxCall) {
         Timber.v("## VOIP: AudioManager startForCall ${mxCall.callId}")
+    }
+
+    private fun setupAudioManager(mxCall: MxCall) {
+        Timber.v("## VOIP: AudioManager setupAudioManager ${mxCall.callId}")
         val audioManager = audioManager ?: return
         savedIsSpeakerPhoneOn = audioManager.isSpeakerphoneOn
         savedIsMicrophoneMute = audioManager.isMicrophoneMute
@@ -150,7 +154,7 @@ class CallAudioManager(
 
     fun onCallConnected(mxCall: MxCall) {
         Timber.v("##VOIP: AudioManager call answered, adjusting current sound device")
-        adjustCurrentSoundDevice(mxCall)
+        setupAudioManager(mxCall)
     }
 
     fun getAvailableSoundDevices(): List<SoundDevice> {

vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt

@@ -88,6 +88,11 @@ class VectorCallViewModel @AssistedInject constructor(
 
     private val currentCallListener = object : WebRtcPeerConnectionManager.CurrentCallListener {
         override fun onCurrentCallChange(call: MxCall?) {
+            // we need to check the state
+            if (call == null) {
+                // we should dismiss, e.g handled by other session?
+                _viewEvents.post(VectorCallViewEvents.DismissNoCall)
+            }
         }
 
         override fun onCaptureStateChanged() {

vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt

@@ -19,10 +19,14 @@ package im.vector.app.features.call
 import android.content.Context
 import android.hardware.camera2.CameraManager
 import androidx.core.content.getSystemService
+import androidx.lifecycle.Lifecycle
+import androidx.lifecycle.LifecycleObserver
+import androidx.lifecycle.OnLifecycleEvent
 import im.vector.app.ActiveSessionDataSource
 import im.vector.app.core.services.BluetoothHeadsetReceiver
 import im.vector.app.core.services.CallService
 import im.vector.app.core.services.WiredHeadsetStateReceiver
+import im.vector.app.push.fcm.FcmHelper
 import io.reactivex.disposables.Disposable
 import io.reactivex.subjects.PublishSubject
 import io.reactivex.subjects.ReplaySubject
@@ -72,7 +76,7 @@ import javax.inject.Singleton
 class WebRtcPeerConnectionManager @Inject constructor(
         private val context: Context,
         private val activeSessionDataSource: ActiveSessionDataSource
-) : CallsListener {
+) : CallsListener, LifecycleObserver {
 
     private val currentSession: Session?
         get() = activeSessionDataSource.currentValue?.orNull()
@@ -170,6 +174,8 @@ class WebRtcPeerConnectionManager @Inject constructor(
 
     private var currentCaptureMode: CaptureFormat = CaptureFormat.HD
 
+    private var isInBackground: Boolean = true
+
     var capturerIsInError = false
         set(value) {
             field = value
@@ -201,6 +207,16 @@ class WebRtcPeerConnectionManager @Inject constructor(
         }
     }
 
+    @OnLifecycleEvent(Lifecycle.Event.ON_RESUME)
+    fun entersForeground() {
+        isInBackground = false
+    }
+
+    @OnLifecycleEvent(Lifecycle.Event.ON_PAUSE)
+    fun entersBackground() {
+        isInBackground = true
+    }
+
     var currentCall: CallContext? = null
         set(value) {
             field = value
@@ -702,6 +718,18 @@ class WebRtcPeerConnectionManager @Inject constructor(
         )
 
         callContext.offerSdp = callInviteContent.offer
+
+        // If this is received while in background, the app will not sync,
+        // and thus won't be able to received events. For example if the call is
+        // accepted on an other session this device will continue ringing
+        if (isInBackground) {
+            if (FcmHelper.isPushSupported()) {
+                // only for push version as fdroid version is already doing it?
+                currentSession?.startAutomaticBackgroundSync(30, 0)
+            } else {
+                // Maybe increase sync freq? but how to set back to default values?
+            }
+        }
     }
 
     private fun createAnswer() {
@@ -849,6 +877,16 @@ class WebRtcPeerConnectionManager @Inject constructor(
         Timber.v("## VOIP onCallManagedByOtherSession: $callId")
         currentCall = null
         CallService.onNoActiveCall(context)
+
+        // did we start background sync? so we should stop it
+        if (isInBackground) {
+            if (FcmHelper.isPushSupported()) {
+                currentSession?.stopAnyBackgroundSync()
+            } else {
+                // for fdroid we should not stop, it should continue syncing
+                // maybe we should restore default timeout/delay though?
+            }
+        }
     }
 
     private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer {

vector/src/main/java/im/vector/app/features/crypto/recover/BootstrapSharedViewModel.kt

@@ -103,7 +103,7 @@ class BootstrapSharedViewModel @AssistedInject constructor(
                     } else {
                         // we need to get existing backup passphrase/key and convert to SSSS
                         val keyVersion = awaitCallback<KeysVersionResult?> {
-                            session.cryptoService().keysBackupService().getVersion(version.version ?: "", it)
+                            session.cryptoService().keysBackupService().getVersion(version.version, it)
                         }
                         if (keyVersion == null) {
                             // strange case... just finish?

vector/src/main/java/im/vector/app/features/rageshake/BugReporter.kt

@@ -212,20 +212,20 @@ class BugReporter @Inject constructor(
                 }
 
                 activeSessionHolder.getSafeActiveSession()
-                            ?.takeIf { !mIsCancelled && withKeyRequestHistory }
+                        ?.takeIf { !mIsCancelled && withKeyRequestHistory }
-                            ?.cryptoService()
+                        ?.cryptoService()
-                            ?.getGossipingEvents()
+                        ?.getGossipingEvents()
-                            ?.let { GossipingEventsSerializer().serialize(it) }
+                        ?.let { GossipingEventsSerializer().serialize(it) }
-                            ?.toByteArray()
+                        ?.toByteArray()
-                            ?.let { rawByteArray ->
+                        ?.let { rawByteArray ->
-                                File(context.cacheDir.absolutePath, KEY_REQUESTS_FILENAME)
+                            File(context.cacheDir.absolutePath, KEY_REQUESTS_FILENAME)
-                                        .also {
+                                    .also {
-                                            it.outputStream()
+                                        it.outputStream()
                                                 .use { os -> os.write(rawByteArray) }
-                                        }
+                                    }
-                            }
+                        }
-                            ?.let { compressFile(it) }
+                        ?.let { compressFile(it) }
-                            ?.let { gzippedFiles.add(it) }
+                        ?.let { gzippedFiles.add(it) }
 
                 var deviceId = "undefined"
                 var userId = "undefined"
@@ -446,6 +446,10 @@ class BugReporter @Inject constructor(
      */
     fun openBugReportScreen(activity: FragmentActivity, forSuggestion: Boolean = false) {
         screenshot = takeScreenshot(activity)
+        activeSessionHolder.getSafeActiveSession()?.let {
+            it.logDbUsageInfo()
+            it.cryptoService().logDbUsageInfo()
+        }
 
         val intent = Intent(activity, BugReportActivity::class.java)
         intent.putExtra("FOR_SUGGESTION", forSuggestion)

vector/src/main/java/im/vector/app/features/rageshake/VectorFileLogger.kt

@@ -102,8 +102,8 @@ class VectorFileLogger @Inject constructor(val context: Context, private val vec
         return if (vectorPreferences.labAllowedExtendedLogging()) {
             false
         } else {
-            // Exclude debug and verbose logs
+            // Exclude verbose logs
-            priority <= Log.DEBUG
+            priority < Log.DEBUG
         }
     }