Rust Migration: use realm migration mechanism

This commit is contained in:
ganfra 2022-05-11 19:20:39 +02:00
parent 88733784cd
commit a2b3839c46
13 changed files with 124 additions and 233 deletions

View file

@ -20,6 +20,7 @@ import android.content.Context
import androidx.test.platform.app.InstrumentationRegistry
import io.realm.Realm
import io.realm.RealmConfiguration
import io.realm.RealmMigration
import org.junit.rules.TemporaryFolder
import org.junit.runner.Description
import org.junit.runners.model.Statement
@ -82,13 +83,16 @@ class TemporaryRealmConfigurationFactory : TemporaryFolder() {
this.tempFolder = tempFolder
}
fun create(realmFilename: String, assetFilename: String? = null, schemaVersion: Long, module: Any?): RealmConfiguration {
fun create(realmFilename: String, assetFilename: String? = null, schemaVersion: Long, module: Any?, migration: RealmMigration? = null): RealmConfiguration {
val configurationBuilder = RealmConfiguration.Builder()
.directory(root)
.name(realmFilename)
.schemaVersion(schemaVersion)
.allowWritesOnUiThread(true)
if (migration != null) {
configurationBuilder.migration(migration)
}
if (module != null) {
configurationBuilder.modules(module)
}

View file

@ -1,66 +0,0 @@
/*
* Copyright 2020 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.migration
import androidx.test.ext.junit.runners.AndroidJUnit4
import io.realm.Realm
import org.amshove.kluent.internal.assertFails
import org.junit.Assert.assertNotNull
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.matrix.android.sdk.InstrumentedTest
import org.matrix.android.sdk.common.TemporaryRealmConfigurationFactory
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.migration.fixtures.rustCryptoStoreMigrationConfiguration
import org.matrix.olm.OlmManager
@RunWith(AndroidJUnit4::class)
class ExtractMigrationDataUseCaseTest : InstrumentedTest {
@Rule
@JvmField
val realmConfigurationFactory = TemporaryRealmConfigurationFactory()
private val extractMigrationData = ExtractMigrationDataUseCase()
@Before
fun setup() {
// Ensure Olm is initialized
OlmManager()
}
@Test
fun given_a_valid_crypto_store_realm_file_then_extraction_should_be_successful() {
val realmConfiguration = realmConfigurationFactory.rustCryptoStoreMigrationConfiguration(populateCryptoStore = true)
val migrationData = Realm.getInstance(realmConfiguration).use {
extractMigrationData(it)
}
assertNotNull(migrationData)
}
@Test
fun given_an_empty_crypto_store_realm_file_then_extraction_should_throw() {
val realmConfiguration = realmConfigurationFactory.rustCryptoStoreMigrationConfiguration(populateCryptoStore = false)
assertFails {
Realm.getInstance(realmConfiguration).use {
extractMigrationData(it)
}
}
}
}

View file

@ -17,9 +17,11 @@
package org.matrix.android.sdk.internal.crypto.store.migration
import androidx.test.ext.junit.runners.AndroidJUnit4
import kotlinx.coroutines.runBlocking
import io.realm.Realm
import org.amshove.kluent.internal.assertEquals
import org.amshove.kluent.internal.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Rule
@ -27,21 +29,17 @@ import org.junit.Test
import org.junit.runner.RunWith
import org.matrix.android.sdk.InstrumentedTest
import org.matrix.android.sdk.common.TemporaryRealmConfigurationFactory
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.migration.fixtures.rustCryptoStoreMigrationConfiguration
import org.matrix.android.sdk.internal.crypto.store.migration.fixtures.configurationForMigrationFrom15To16
import org.matrix.olm.OlmManager
import uniffi.olm.OlmMachine
import java.util.concurrent.CountDownLatch
@RunWith(AndroidJUnit4::class)
class RustCryptoStoreMigrateUseCaseTest : InstrumentedTest {
class RealmMigrateCryptoTo016Test : InstrumentedTest {
@Rule
@JvmField
val realmConfigurationFactory = TemporaryRealmConfigurationFactory()
private val extractMigrationData = ExtractMigrationDataUseCase()
@Before
fun setup() {
// Ensure Olm is initialized
@ -49,15 +47,11 @@ class RustCryptoStoreMigrateUseCaseTest : InstrumentedTest {
}
@Test
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() = runBlocking {
val realmConfiguration = realmConfigurationFactory.rustCryptoStoreMigrationConfiguration(populateCryptoStore = true)
val cryptoStoreMigrate = RustCryptoStoreMigrateUseCase(realmConfiguration, realmConfigurationFactory.root, extractMigrationData)
val latch = CountDownLatch(1)
val progressListener = ProgressListener(latch)
val result = cryptoStoreMigrate(progressListener)
latch.await()
assert(result.isSuccess)
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = true)
Realm.getInstance(realmConfiguration).use {
assertTrue(it.isEmpty)
}
val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
assertEquals("mW7LWO4zmhH8Ttuvmzn27vm/USXSKBPgmg7FKQITLiU", machine.identityKeys()["ed25519"])
assertNotNull(machine.getBackupKeys())
@ -67,20 +61,18 @@ class RustCryptoStoreMigrateUseCaseTest : InstrumentedTest {
assertTrue(crossSigningStatus.hasUserSigning)
}
@Test
fun given_an_empty_crypto_store_realm_file_then_migration_should_fail() = runBlocking {
val realmConfiguration = realmConfigurationFactory.rustCryptoStoreMigrationConfiguration(populateCryptoStore = false)
val cryptoStoreMigrate = RustCryptoStoreMigrateUseCase(realmConfiguration, realmConfigurationFactory.root, extractMigrationData)
val progressListener = ProgressListener()
val result = cryptoStoreMigrate(progressListener)
assert(result.isFailure)
}
private class ProgressListener(val latch: CountDownLatch? = null) : uniffi.olm.ProgressListener {
override fun onProgress(progress: Int, total: Int) {
if (progress == total) {
latch?.countDown()
}
}
@Test
fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
Realm.getInstance(realmConfiguration).use {
assertTrue(it.isEmpty)
}
val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
assertNull(machine.getBackupKeys())
val crossSigningStatus = machine.crossSigningStatus()
assertFalse(crossSigningStatus.hasMaster)
assertFalse(crossSigningStatus.hasSelfSigning)
assertFalse(crossSigningStatus.hasUserSigning)
}
}

View file

@ -18,13 +18,15 @@ package org.matrix.android.sdk.internal.crypto.store.migration.fixtures
import io.realm.RealmConfiguration
import org.matrix.android.sdk.common.TemporaryRealmConfigurationFactory
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
fun TemporaryRealmConfigurationFactory.rustCryptoStoreMigrationConfiguration(populateCryptoStore: Boolean): RealmConfiguration {
fun TemporaryRealmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore: Boolean): RealmConfiguration {
return create(
realmFilename = "crypto_store_rust_migration.realm",
assetFilename = "crypto_store_rust_migration.realm".takeIf { populateCryptoStore },
schemaVersion = 15L,
module = RealmCryptoStoreModule()
realmFilename = "crypto_store.realm",
assetFilename = "crypto_store_migration_15_to_16.realm".takeIf { populateCryptoStore },
schemaVersion = 16L,
module = RealmCryptoStoreModule(),
migration = RealmCryptoStoreMigration(root)
)
}

View file

@ -147,37 +147,6 @@ internal class RealmCryptoStore @Inject constructor(
.setWriteAsyncExecutor(monarchyWriteAsyncExecutor)
.build()
init {
// Ensure CryptoMetadataEntity is inserted in DB
doRealmTransaction(realmConfiguration) { realm ->
var currentMetadata = realm.where<CryptoMetadataEntity>().findFirst()
var deleteAll = false
if (currentMetadata != null) {
// Check credentials
// The device id may not have been provided in credentials.
// Check it only if provided, else trust the stored one.
if (currentMetadata.userId != userId ||
(deviceId != null && deviceId != currentMetadata.deviceId)) {
Timber.w("## open() : Credentials do not match, close this store and delete data")
deleteAll = true
currentMetadata = null
}
}
if (currentMetadata == null) {
if (deleteAll) {
realm.deleteAll()
}
// Metadata not found, or database cleaned, create it
realm.createObject(CryptoMetadataEntity::class.java, userId).apply {
deviceId = this@RealmCryptoStore.deviceId
}
}
}
}
/* ==========================================================================================
* Other data
* ========================================================================================== */

View file

@ -33,10 +33,13 @@ import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016
import org.matrix.android.sdk.internal.di.SessionFilesDirectory
import timber.log.Timber
import java.io.File
import javax.inject.Inject
internal class RealmCryptoStoreMigration @Inject constructor() : RealmMigration {
internal class RealmCryptoStoreMigration @Inject constructor(@SessionFilesDirectory private val sessionFilesDirectory: File) : RealmMigration {
/**
* Forces all RealmCryptoStoreMigration instances to be equal
* Avoids Realm throwing when multiple instances of the migration are set
@ -47,7 +50,7 @@ internal class RealmCryptoStoreMigration @Inject constructor() : RealmMigration
// 0, 1, 2: legacy Riot-Android
// 3: migrate to RiotX schema
// 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6)
val schemaVersion = 15L
val schemaVersion = 16L
override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
Timber.d("Migrating Realm Crypto from $oldVersion to $newVersion")
@ -67,5 +70,6 @@ internal class RealmCryptoStoreMigration @Inject constructor() : RealmMigration
if (oldVersion < 13) MigrateCryptoTo013(realm).perform()
if (oldVersion < 14) MigrateCryptoTo014(realm).perform()
if (oldVersion < 15) MigrateCryptoTo015(realm).perform()
if (oldVersion < 16) MigrateCryptoTo016(realm, sessionFilesDirectory).perform()
}
}

View file

@ -0,0 +1,45 @@
/*
* Copyright (c) 2022 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db.migration
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase
import org.matrix.android.sdk.internal.util.database.RealmMigrator
import timber.log.Timber
import uniffi.olm.ProgressListener
import java.io.File
class MigrateCryptoTo016(realm: DynamicRealm, private val sessionFilesDirectory: File) : RealmMigrator(realm, 16) {
private val extractMigrationData = ExtractMigrationDataUseCase()
override fun doMigrate(realm: DynamicRealm) {
val progressListener = object : ProgressListener {
override fun onProgress(progress: Int, total: Int) {
Timber.v("OnProgress: $progress/$total")
}
}
try {
val migrationData = extractMigrationData(realm)
uniffi.olm.migrate(migrationData, sessionFilesDirectory.path, null, progressListener)
} catch (failure: Throwable) {
Timber.e(failure, "Failure while calling rust migration method")
throw failure
}
realm.deleteAll()
}
}

View file

@ -14,14 +14,14 @@
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.migration
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
import io.realm.Realm
import io.realm.kotlin.where
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import io.realm.DynamicRealm
import io.realm.DynamicRealmObject
import org.matrix.android.sdk.internal.crypto.model.OlmInboundGroupSessionWrapper2
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmSession
import org.matrix.olm.OlmUtility
import uniffi.olm.CrossSigningKeyExport
import uniffi.olm.MigrationData
@ -29,13 +29,12 @@ import uniffi.olm.PickledAccount
import uniffi.olm.PickledInboundGroupSession
import uniffi.olm.PickledSession
import java.nio.charset.Charset
import javax.inject.Inject
private val charset = Charset.forName("UTF-8")
internal class ExtractMigrationDataUseCase @Inject constructor() {
internal class ExtractMigrationDataUseCase {
operator fun invoke(realm: Realm): MigrationData {
operator fun invoke(realm: DynamicRealm): MigrationData {
return try {
extract(realm) ?: throw ExtractMigrationDataFailure
} catch (failure: Throwable) {
@ -43,32 +42,33 @@ internal class ExtractMigrationDataUseCase @Inject constructor() {
}
}
private fun extract(realm: Realm): MigrationData? {
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst() ?: return null
private fun extract(realm: DynamicRealm): MigrationData? {
val metadataEntity = realm.where("CryptoMetadataEntity").findFirst() ?: return null
val pickleKey = OlmUtility.getRandomKey()
val olmSessionEntities = realm.where<OlmSessionEntity>().findAll()
val olmSessionEntities = realm.where("OlmSessionEntity").findAll()
val pickledSessions = olmSessionEntities.map { it.toPickledSession(pickleKey) }
val inboundGroupSessionEntities = realm.where<OlmInboundGroupSessionEntity>().findAll()
val inboundGroupSessionEntities = realm.where("OlmInboundGroupSessionEntity").findAll()
val pickledInboundGroupSessions = inboundGroupSessionEntities.map { it.toPickledInboundGroupSession(pickleKey) }
val masterKey = metadataEntity.xSignMasterPrivateKey
val userKey = metadataEntity.xSignUserPrivateKey
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
val masterKey = metadataEntity.getString("xSignMasterPrivateKey")
val userKey = metadataEntity.getString("xSignUserPrivateKey")
val selfSignedKey = metadataEntity.getString("xSignSelfSignedPrivateKey")
val userId = metadataEntity.userId!!
val deviceId = metadataEntity.deviceId!!
val backupVersion = metadataEntity.backupVersion
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
val userId = metadataEntity.getString("userId")!!
val deviceId = metadataEntity.getString("deviceId")!!
val backupVersion = metadataEntity.getString("backupVersion")
val backupRecoveryKey = metadataEntity.getString("keyBackupRecoveryKey")
val trackedUserEntities = realm.where<UserEntity>().findAll()
val trackedUserEntities = realm.where("UserEntity").findAll()
val trackedUserIds = trackedUserEntities.mapNotNull {
it.userId
it.getString("userId")
}
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
val isOlmAccountShared = metadataEntity.getBoolean("deviceKeysSentToServer")
val olmAccount = metadataEntity.getOlmAccount()!!
val olmAccountStr = metadataEntity.getString("olmAccountData")
val olmAccount = deserializeFromRealm<OlmAccount>(olmAccountStr)!!
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
val pickledAccount = PickledAccount(
userId = userId,
@ -93,9 +93,11 @@ internal class ExtractMigrationDataUseCase @Inject constructor() {
)
}
private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession {
val senderKey = this.senderKey ?: ""
val olmInboundGroupSession = getInboundGroupSession()!!
private fun DynamicRealmObject.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession {
val senderKey = this.getString("senderKey") ?: ""
val backedUp = this.getBoolean("backedUp")
val olmInboundGroupSessionStr = this.getString("olmInboundGroupSessionData")
val olmInboundGroupSession = deserializeFromRealm<OlmInboundGroupSessionWrapper2>(olmInboundGroupSessionStr)!!
val pickledInboundGroupSession = olmInboundGroupSession.olmInboundGroupSession!!.pickle(pickleKey, StringBuffer()).asString()
return PickledInboundGroupSession(
pickle = pickledInboundGroupSession,
@ -108,10 +110,11 @@ internal class ExtractMigrationDataUseCase @Inject constructor() {
)
}
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.deviceKey ?: ""
val lastReceivedMessageTs = this.lastReceivedMessageTs
val olmSession = getOlmSession()!!
private fun DynamicRealmObject.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.getString("deviceKey") ?: ""
val lastReceivedMessageTs = this.getLong("lastReceivedMessageTs")
val olmSessionStr = this.getString("olmSessionData")
val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
return PickledSession(
pickle = pickledOlmSession,

View file

@ -14,6 +14,6 @@
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.migration
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
object ExtractMigrationDataFailure : java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.")

View file

@ -1,20 +0,0 @@
/*
* Copyright (c) 2022 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.migration
class CleanUpCryptoStoreUseCase {
}

View file

@ -1,43 +0,0 @@
/*
* Copyright (c) 2022 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.migration
import io.realm.Realm
import io.realm.RealmConfiguration
import org.matrix.android.sdk.internal.database.awaitTransaction
import org.matrix.android.sdk.internal.di.CryptoDatabase
import org.matrix.android.sdk.internal.di.SessionFilesDirectory
import uniffi.olm.ProgressListener
import java.io.File
import javax.inject.Inject
internal class RustCryptoStoreMigrateUseCase @Inject constructor(
@CryptoDatabase private val realmConfiguration: RealmConfiguration,
@SessionFilesDirectory private val dataDir: File,
private val extractMigrationData: ExtractMigrationDataUseCase) {
suspend operator fun invoke(progressListener: ProgressListener) = runCatching {
migrate(progressListener)
}
private suspend fun migrate(progressListener: ProgressListener) {
awaitTransaction(realmConfiguration) { realm: Realm ->
val migrationData = extractMigrationData(realm)
uniffi.olm.migrate(migrationData, dataDir.path, null, progressListener)
}
}
}

View file

@ -43,7 +43,6 @@ internal class DefaultLegacySessionImporter @Inject constructor(
private val context: Context,
private val sessionParamsStore: SessionParamsStore,
private val realmKeysUtils: RealmKeysUtils,
private val realmCryptoStoreMigration: RealmCryptoStoreMigration
) : LegacySessionImporter {
private val loginStorage = LoginStorage(context)
@ -165,6 +164,8 @@ internal class DefaultLegacySessionImporter @Inject constructor(
newLocation.deleteRecursively()
newLocation.mkdirs()
val realmCryptoStoreMigration = RealmCryptoStoreMigration(newLocation)
Timber.d("Migration: create legacy realm configuration")
val realmConfiguration = RealmConfiguration.Builder()