XSigning keys: use json instead of object serialization

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt

@@ -112,6 +112,7 @@ internal abstract class CryptoModule {
         @SessionScope
         fun providesRealmConfiguration(@SessionFilesDirectory directory: File,
                                        @UserMd5 userMd5: String,
+                                       realmCryptoStoreMigration: RealmCryptoStoreMigration,
                                        realmKeysUtils: RealmKeysUtils): RealmConfiguration {
             return RealmConfiguration.Builder()
                     .directory(directory)
@@ -121,7 +122,7 @@ internal abstract class CryptoModule {
                     .name("crypto_store.realm")
                     .modules(RealmCryptoStoreModule())
                     .schemaVersion(RealmCryptoStoreMigration.CRYPTO_STORE_SCHEMA_VERSION)
-                    .migration(RealmCryptoStoreMigration)
+                    .migration(realmCryptoStoreMigration)
                     .build()
         }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/Helper.kt

@@ -62,6 +62,7 @@ fun doRealmTransaction(realmConfiguration: RealmConfiguration, action: (Realm) -
         realm.executeTransaction { action.invoke(it) }
     }
 }
+
 fun doRealmTransactionAsync(realmConfiguration: RealmConfiguration, action: (Realm) -> Unit) {
     Realm.getInstance(realmConfiguration).use { realm ->
         realm.executeTransactionAsync { action.invoke(it) }
@@ -79,31 +80,26 @@ fun serializeForRealm(o: Any?): String? {
     val baos = ByteArrayOutputStream()
     val gzis = CompatUtil.createGzipOutputStream(baos)
     val out = ObjectOutputStream(gzis)
-
+    out.use {
-    out.writeObject(o)
+        it.writeObject(o)
-    out.close()
+    }
-
     return Base64.encodeToString(baos.toByteArray(), Base64.DEFAULT)
 }
 
 /**
  * Do the opposite of serializeForRealm.
  */
+@Suppress("UNCHECKED_CAST")
 fun <T> deserializeFromRealm(string: String?): T? {
     if (string == null) {
         return null
     }
-
     val decodedB64 = Base64.decode(string.toByteArray(), Base64.DEFAULT)
 
     val bais = ByteArrayInputStream(decodedB64)
     val gzis = GZIPInputStream(bais)
     val ois = ObjectInputStream(gzis)
-
+    return ois.use {
-    @Suppress("UNCHECKED_CAST")
+        it.readObject() as T
-    val result = ois.readObject() as T
+    }
-
-    ois.close()
-
-    return result
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStore.kt

@@ -36,7 +36,6 @@ import im.vector.matrix.android.internal.crypto.OutgoingGossipingRequestState
 import im.vector.matrix.android.internal.crypto.OutgoingRoomKeyRequest
 import im.vector.matrix.android.internal.crypto.OutgoingSecretRequest
 import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
-import im.vector.matrix.android.internal.crypto.crosssigning.DeviceTrustLevel
 import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
 import im.vector.matrix.android.internal.crypto.model.CryptoDeviceInfo
 import im.vector.matrix.android.internal.crypto.model.OlmInboundGroupSessionWrapper
@@ -46,6 +45,7 @@ import im.vector.matrix.android.internal.crypto.model.toEntity
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import im.vector.matrix.android.internal.crypto.store.PrivateKeysInfo
 import im.vector.matrix.android.internal.crypto.store.SavedKeyBackupKeyInfo
+import im.vector.matrix.android.internal.crypto.store.db.mapper.CrossSigningKeysMapper
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.CryptoMapper
@@ -91,6 +91,7 @@ import kotlin.collections.set
 @SessionScope
 internal class RealmCryptoStore @Inject constructor(
         @CryptoDatabase private val realmConfiguration: RealmConfiguration,
+        private val crossSigningKeysMapper: CrossSigningKeysMapper,
         private val credentials: Credentials) : IMXCryptoStore {
 
     /* ==========================================================================================
@@ -309,36 +310,19 @@ internal class RealmCryptoStore @Inject constructor(
                         } else {
                             CrossSigningInfoEntity.getOrCreate(realm, userId).let { signingInfo ->
                                 // What should we do if we detect a change of the keys?
-
                                 val existingMaster = signingInfo.getMasterKey()
                                 if (existingMaster != null && existingMaster.publicKeyBase64 == masterKey.unpaddedBase64PublicKey) {
-                                    // update signatures?
+                                    crossSigningKeysMapper.update(existingMaster, masterKey)
-                                    existingMaster.putSignatures(masterKey.signatures)
-                                    existingMaster.usages = masterKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                            ?: RealmList()
                                 } else {
-                                    val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                    val keyEntity = crossSigningKeysMapper.map(masterKey)
-                                        this.publicKeyBase64 = masterKey.unpaddedBase64PublicKey
-                                        this.usages = masterKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
-                                        this.putSignatures(masterKey.signatures)
-                                    }
                                     signingInfo.setMasterKey(keyEntity)
                                 }
 
                                 val existingSelfSigned = signingInfo.getSelfSignedKey()
                                 if (existingSelfSigned != null && existingSelfSigned.publicKeyBase64 == selfSigningKey.unpaddedBase64PublicKey) {
-                                    // update signatures?
+                                    crossSigningKeysMapper.update(existingSelfSigned, selfSigningKey)
-                                    existingSelfSigned.putSignatures(selfSigningKey.signatures)
-                                    existingSelfSigned.usages = selfSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                            ?: RealmList()
                                 } else {
-                                    val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                    val keyEntity = crossSigningKeysMapper.map(selfSigningKey)
-                                        this.publicKeyBase64 = selfSigningKey.unpaddedBase64PublicKey
-                                        this.usages = selfSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
-                                        this.putSignatures(selfSigningKey.signatures)
-                                    }
                                     signingInfo.setSelfSignedKey(keyEntity)
                                 }
 
@@ -346,21 +330,12 @@ internal class RealmCryptoStore @Inject constructor(
                                 if (userSigningKey != null) {
                                     val existingUSK = signingInfo.getUserSigningKey()
                                     if (existingUSK != null && existingUSK.publicKeyBase64 == userSigningKey.unpaddedBase64PublicKey) {
-                                        // update signatures?
+                                        crossSigningKeysMapper.update(existingUSK, userSigningKey)
-                                        existingUSK.putSignatures(userSigningKey.signatures)
-                                        existingUSK.usages = userSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
                                     } else {
-                                        val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                        val keyEntity = crossSigningKeysMapper.map(userSigningKey)
-                                            this.publicKeyBase64 = userSigningKey.unpaddedBase64PublicKey
-                                            this.usages = userSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                    ?: RealmList()
-                                            this.putSignatures(userSigningKey.signatures)
-                                        }
                                         signingInfo.setUserSignedKey(keyEntity)
                                     }
                                 }
-
                                 userEntity.crossSigningInfoEntity = signingInfo
                             }
                         }
@@ -1183,9 +1158,9 @@ internal class RealmCryptoStore @Inject constructor(
      * Cross Signing
      * ========================================================================================== */
     override fun getMyCrossSigningInfo(): MXCrossSigningInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            it.where<CryptoMetadataEntity>().findFirst()
+            it.where<CryptoMetadataEntity>().findFirst()?.userId
-        }?.userId?.let {
+        }?.let {
             getCrossSigningInfo(it)
         }
     }
@@ -1304,33 +1279,24 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getCrossSigningInfo(userId: String): MXCrossSigningInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) { realm ->
+        return doWithRealm(realmConfiguration) { realm ->
-            realm.where(CrossSigningInfoEntity::class.java)
+            val crossSigningInfo = realm.where(CrossSigningInfoEntity::class.java)
                     .equalTo(CrossSigningInfoEntityFields.USER_ID, userId)
                     .findFirst()
-        }?.let { xsignInfo ->
+            if (crossSigningInfo == null) {
-            mapCrossSigningInfoEntity(xsignInfo)
+                null
+            } else {
+                mapCrossSigningInfoEntity(crossSigningInfo)
+            }
         }
     }
 
     private fun mapCrossSigningInfoEntity(xsignInfo: CrossSigningInfoEntity): MXCrossSigningInfo {
+        val userId = xsignInfo.userId ?: ""
         return MXCrossSigningInfo(
-                userId = xsignInfo.userId ?: "",
+                userId = userId,
                 crossSigningKeys = xsignInfo.crossSigningKeys.mapNotNull {
-                    val pubKey = it.publicKeyBase64 ?: return@mapNotNull null
+                    crossSigningKeysMapper.map(userId, it)
-                    CryptoCrossSigningKey(
-                            userId = xsignInfo.userId ?: "",
-                            keys = mapOf("ed25519:$pubKey" to pubKey),
-                            usages = it.usages.map { it },
-                            signatures = it.getSignatures(),
-                            trustLevel = it.trustLevelEntity?.let {
-                                DeviceTrustLevel(
-                                        crossSigningVerified = it.crossSignedVerified ?: false,
-                                        locallyVerified = it.locallyVerified ?: false
-                                )
-                            }
-
-                    )
                 }
         )
     }
@@ -1341,26 +1307,7 @@ internal class RealmCryptoStore @Inject constructor(
                     realm.where<CrossSigningInfoEntity>()
                             .equalTo(UserEntityFields.USER_ID, userId)
                 },
-                { entity ->
+                { mapCrossSigningInfoEntity(it) }
-                    MXCrossSigningInfo(
-                            userId = userId,
-                            crossSigningKeys = entity.crossSigningKeys.mapNotNull {
-                                val pubKey = it.publicKeyBase64 ?: return@mapNotNull null
-                                CryptoCrossSigningKey(
-                                        userId = userId,
-                                        keys = mapOf("ed25519:$pubKey" to pubKey),
-                                        usages = it.usages.map { it },
-                                        signatures = it.getSignatures(),
-                                        trustLevel = it.trustLevelEntity?.let {
-                                            DeviceTrustLevel(
-                                                    crossSigningVerified = it.crossSignedVerified ?: false,
-                                                    locallyVerified = it.locallyVerified ?: false
-                                            )
-                                        }
-                                )
-                            }
-                    )
-                }
         )
         return Transformations.map(liveData) {
             it.firstOrNull().toOptional()
@@ -1402,17 +1349,8 @@ internal class RealmCryptoStore @Inject constructor(
             // existing.crossSigningKeys.forEach { it.deleteFromRealm() }
             val xkeys = RealmList<KeyInfoEntity>()
             info.crossSigningKeys.forEach { cryptoCrossSigningKey ->
-                xkeys.add(
+                val keyEntity = crossSigningKeysMapper.map(cryptoCrossSigningKey)
-                        realm.createObject(KeyInfoEntity::class.java).also { keyInfoEntity ->
+                xkeys.add(keyEntity)
-                            keyInfoEntity.publicKeyBase64 = cryptoCrossSigningKey.unpaddedBase64PublicKey
-                            keyInfoEntity.usages = cryptoCrossSigningKey.usages?.let { RealmList(*it.toTypedArray()) }
-                                    ?: RealmList()
-                            keyInfoEntity.putSignatures(cryptoCrossSigningKey.signatures)
-                            // TODO how to handle better, check if same keys?
-                            // reset trust
-                            keyInfoEntity.trustLevelEntity = null
-                        }
-                )
             }
             existing.crossSigningKeys = xkeys
         }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -20,6 +20,7 @@ import com.squareup.moshi.Moshi
 import com.squareup.moshi.Types
 import im.vector.matrix.android.api.util.JsonDict
 import im.vector.matrix.android.internal.crypto.model.MXDeviceInfo
+import im.vector.matrix.android.internal.crypto.store.db.mapper.CrossSigningKeysMapper
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.CryptoMetadataEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.DeviceInfoEntityFields
@@ -33,11 +34,14 @@ import im.vector.matrix.android.internal.di.SerializeNulls
 import io.realm.DynamicRealm
 import io.realm.RealmMigration
 import timber.log.Timber
+import javax.inject.Inject
 
-internal object RealmCryptoStoreMigration : RealmMigration {
+internal class RealmCryptoStoreMigration @Inject constructor(private val crossSigningKeysMapper: CrossSigningKeysMapper) : RealmMigration {
 
     // Version 1L added Cross Signing info persistence
-    const val CRYPTO_STORE_SCHEMA_VERSION = 3L
+    companion object {
+        const val CRYPTO_STORE_SCHEMA_VERSION = 4L
+    }
 
     override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
         Timber.v("Migrating Realm Crypto from $oldVersion to $newVersion")
@@ -45,6 +49,7 @@ internal object RealmCryptoStoreMigration : RealmMigration {
         if (oldVersion <= 0) migrateTo1(realm)
         if (oldVersion <= 1) migrateTo2(realm)
         if (oldVersion <= 2) migrateTo3(realm)
+        if (oldVersion <= 3) migrateTo4(realm)
     }
 
     private fun migrateTo1(realm: DynamicRealm) {
@@ -193,4 +198,19 @@ internal object RealmCryptoStoreMigration : RealmMigration {
                 ?.addField(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY, String::class.java)
                 ?.addField(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY_VERSION, String::class.java)
     }
+
+    private fun migrateTo4(realm: DynamicRealm) {
+        Timber.d("Updating KeyInfoEntity table")
+        val keyInfoEntities = realm.where("KeyInfoEntity").findAll()
+        try {
+            keyInfoEntities.forEach {
+                val stringSignatures = it.getString(KeyInfoEntityFields.SIGNATURES)
+                val objectSignatures: Map<String, Map<String, String>>? = deserializeFromRealm(stringSignatures)
+                val jsonSignatures = crossSigningKeysMapper.serializeSignatures(objectSignatures)
+                it.setString(KeyInfoEntityFields.SIGNATURES, jsonSignatures)
+            }
+        } catch (failure: Throwable) {
+
+        }
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/mapper/CrossSigningKeysMapper.kt

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.mapper
+
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.Types
+import im.vector.matrix.android.internal.crypto.crosssigning.DeviceTrustLevel
+import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
+import im.vector.matrix.android.internal.crypto.store.db.model.KeyInfoEntity
+import io.realm.RealmList
+import timber.log.Timber
+import javax.inject.Inject
+
+internal class CrossSigningKeysMapper @Inject constructor(moshi: Moshi) {
+
+    private val signaturesAdapter = moshi.adapter<Map<String, Map<String, String>>>(Types.newParameterizedType(
+            Map::class.java,
+            String::class.java,
+            Any::class.java
+    ))
+
+    fun update(keyInfo: KeyInfoEntity, cryptoCrossSigningKey: CryptoCrossSigningKey) {
+        // update signatures?
+        keyInfo.signatures = serializeSignatures(cryptoCrossSigningKey.signatures)
+        keyInfo.usages = cryptoCrossSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
+                ?: RealmList()
+    }
+
+    fun map(userId: String?, keyInfo: KeyInfoEntity?): CryptoCrossSigningKey? {
+        val pubKey = keyInfo?.publicKeyBase64 ?: return null
+        return CryptoCrossSigningKey(
+                userId = userId ?: "",
+                keys = mapOf("ed25519:$pubKey" to pubKey),
+                usages = keyInfo.usages.map { it },
+                signatures = deserializeSignatures(keyInfo.signatures),
+                trustLevel = keyInfo.trustLevelEntity?.let {
+                    DeviceTrustLevel(
+                            crossSigningVerified = it.crossSignedVerified ?: false,
+                            locallyVerified = it.locallyVerified ?: false
+                    )
+                }
+        )
+    }
+
+    fun map(keyInfo: CryptoCrossSigningKey): KeyInfoEntity {
+        return KeyInfoEntity().apply {
+            publicKeyBase64 = keyInfo.unpaddedBase64PublicKey
+            usages = keyInfo.usages?.let { RealmList(*it.toTypedArray()) } ?: RealmList()
+            signatures = serializeSignatures(keyInfo.signatures)
+            // TODO how to handle better, check if same keys?
+            // reset trust
+            trustLevelEntity = null
+        }
+    }
+
+
+    fun serializeSignatures(signatures: Map<String, Map<String, String>>?): String {
+        return signaturesAdapter.toJson(signatures)
+    }
+
+    fun deserializeSignatures(signatures: String?): Map<String, Map<String, String>>? {
+        if (signatures == null) {
+            return null
+        }
+        return try {
+            signaturesAdapter.fromJson(signatures)
+        } catch (failure: Throwable) {
+            Timber.e(failure)
+            null
+        }
+    }
+
+
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/model/KeyInfoEntity.kt

@@ -16,8 +16,6 @@
 
 package im.vector.matrix.android.internal.crypto.store.db.model
 
-import im.vector.matrix.android.internal.crypto.store.db.deserializeFromRealm
-import im.vector.matrix.android.internal.crypto.store.db.serializeForRealm
 import io.realm.RealmList
 import io.realm.RealmObject
 
@@ -31,15 +29,4 @@ internal open class KeyInfoEntity(
          */
         var signatures: String? = null,
         var trustLevelEntity: TrustLevelEntity? = null
-) : RealmObject() {
+) : RealmObject()
-
-    // Deserialize data
-    fun getSignatures(): Map<String, Map<String, String>>? {
-        return deserializeFromRealm(signatures)
-    }
-
-    // Serialize data
-    fun putSignatures(deviceInfo: Map<String, Map<String, String>>?) {
-        signatures = serializeForRealm(deviceInfo)
-    }
-}