Merge branch 'develop' into rebranding_rebase

* develop: (123 commits)
  Fixes #1647 share not working
  Put xmx to 2048m
  Update changelog after PR merged
  Version++
  Prepare release 0.91.04-beta
  Add the new value to the ViewEvent, because the state is maybe not up to date.
  Fix lint error, following the upgrade of the libs
  Revert to gradle build 3.5.3
  Clean code
  Fix leaving selected group
  QuickFix / crash when  starting in airplane mode
  Group: rework a bit how and when we fetch data about groups
  EventInsert: add InsertType to avoid trying to process events we shouldn't
  Upload device keys only once to the homeserver and fix crash when no network (#1629)
  Upgrade some dependencies
  Add a delay to avoid crash. Sounds like a workaround...
  Handle certificate error in case of Direct Login
  Handle JobCancellationException
  Simplify the server selection screen: remove the "Continue" button
  Re-activate Wellknown support with updated UI (#1614)
  ...

# Conflicts:
#	vector/build.gradle
#	vector/src/main/res/values/strings.xml

CHANGES.md

@@ -1,6 +1,60 @@
-Changes in RiotX 0.23.0 (2020-XX-XX)
+Changes in Riot.imX 0.91.5 (2020-XX-XX)
 ===================================================
 
+Features ✨:
+ -
+
+Improvements 🙌:
+ - Cleaning chunks with lots of events as long as a threshold has been exceeded (35_000 events in DB) (#1634)
+ - Creating and listening to EventInsertEntity. (#1634)
+ - Handling (almost) properly the groups fetching (#1634)
+
+Bugfix 🐛:
+ - Regression |  Share action menu do not work (#1647)
+
+Translations 🗣:
+ -
+
+SDK API changes ⚠️:
+ - 
+
+Build 🧱:
+ - Upgrade some dependencies
+ - Revert to build-tools 3.5.3
+
+Other changes:
+ -
+
+Changes in Riot.imX 0.91.4 (2020-07-06)
+===================================================
+
+Features ✨:
+ - Re-activate Wellknown support with updated UI (#1614)
+
+Improvements 🙌:
+ - Upload device keys only once to the homeserver and fix crash when no network (#1629)
+
+Bugfix 🐛:
+ - Fix crash when coming from a notification (#1601)
+ - Fix Exception when importing keys (#1576)
+ - File isn't downloaded when another file with the same name already exists (#1578)
+ - saved images don't show up in gallery (#1324)
+ - Fix reply fallback leaking sender locale (#429)
+
+Build 🧱:
+ - Fix lint false-positive about WorkManager (#1012)
+ - Upgrade build-tools from 3.5.3 to 3.6.3
+ - Upgrade gradle from 5.4.1 to 5.6.4
+
+Changes in Riot.imX 0.91.3 (2020-07-01)
+===================================================
+
+Notes:
+ - This version is the third beta version of RiotX codebase published as Riot-Android on the PlayStore.
+ - Changelog below includes changes of v0.91.0, v0.91.1, and v0.91.2, because the first beta versions have been tagged and
+ published from the branch feature/migration_from_legacy.
+ - This version uses temporary name `Riot.imX`, to distinguish the app with RiotX app.
+
 Features ✨:
  - Call with WebRTC support (##611)
  - Add capability to change the display name (#1529)
@@ -9,6 +63,12 @@ Improvements 🙌:
  - "Add Matrix app" menu is now always visible (#1495)
  - Handle `/op`, `/deop`, and `/nick` commands (#12)
  - Prioritising Recovery key over Recovery passphrase (#1463)
+ - Room Settings: Name, Topic, Photo, Aliases, History Visibility (#1455)
+ - Update user avatar (#1054)
+ - Allow self-signed certificate (#1564)
+ - Improve file download and open in timeline
+ - Catchup tab is removed temporarily (#1565)
+ - Render room avatar change (#1319)
 
 Bugfix 🐛:
  - Fix dark theme issue on login screen (#1097)
@@ -16,12 +76,8 @@ Bugfix 🐛:
  - User could not redact message that they have sent (#1543)
  - Use vendor prefix for non merged MSC (#1537)
  - Compress images before sending (#1333)
-
-Translations 🗣:
- -
-
-SDK API changes ⚠️:
- - 
+ - Searching by displayname is case sensitive (#1468)
+ - Fix layout overlap issue (#1407)
 
 Build 🧱:
  - Enable code optimization (Proguard)

build.gradle

@@ -1,7 +1,7 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 
 buildscript {
-    ext.kotlin_version = '1.3.50'
+    ext.kotlin_version = '1.3.72'
     repositories {
         google()
         jcenter()
@@ -10,12 +10,13 @@ buildscript {
         }
     }
     dependencies {
+        // Warning: 3.6.3 leads to infinite gradle builds. Stick to 3.5.3 for the moment
         classpath 'com.android.tools.build:gradle:3.5.3'
         classpath 'com.google.gms:google-services:4.3.2'
         classpath "com.airbnb.okreplay:gradle-plugin:1.5.0"
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath 'org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.7.1'
-        classpath 'com.google.android.gms:oss-licenses-plugin:0.9.5'
+        classpath 'com.google.android.gms:oss-licenses-plugin:0.10.2'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files

gradle.properties

@@ -8,7 +8,7 @@
 # The setting is particularly useful for tweaking memory settings.
 android.enableJetifier=true
 android.useAndroidX=true
-org.gradle.jvmargs=-Xmx8192m
+org.gradle.jvmargs=-Xmx2048m
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit
 # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Fri Sep 27 10:10:35 CEST 2019
+#Thu Jul 02 12:33:07 CEST 2020
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-all.zip

matrix-sdk-android-rx/build.gradle

@@ -39,7 +39,7 @@ dependencies {
     implementation 'io.reactivex.rxjava2:rxkotlin:2.3.0'
     implementation 'io.reactivex.rxjava2:rxandroid:2.1.1'
     // Paging
-    implementation "androidx.paging:paging-runtime-ktx:2.1.0"
+    implementation "androidx.paging:paging-runtime-ktx:2.1.2"
 
     // Logging
     implementation 'com.jakewharton.timber:timber:4.7.1'

matrix-sdk-android-rx/src/main/java/im/vector/matrix/rx/RxRoom.kt

@@ -16,12 +16,14 @@
 
 package im.vector.matrix.rx
 
+import android.net.Uri
 import im.vector.matrix.android.api.query.QueryStringValue
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.room.Room
 import im.vector.matrix.android.api.session.room.members.RoomMemberQueryParams
 import im.vector.matrix.android.api.session.room.model.EventAnnotationsSummary
 import im.vector.matrix.android.api.session.room.model.ReadReceipt
+import im.vector.matrix.android.api.session.room.model.RoomHistoryVisibility
 import im.vector.matrix.android.api.session.room.model.RoomMemberSummary
 import im.vector.matrix.android.api.session.room.model.RoomSummary
 import im.vector.matrix.android.api.session.room.notification.RoomNotificationState
@@ -101,6 +103,30 @@ class RxRoom(private val room: Room) {
     fun invite(userId: String, reason: String? = null): Completable = completableBuilder<Unit> {
         room.invite(userId, reason, it)
     }
+
+    fun updateTopic(topic: String): Completable = completableBuilder<Unit> {
+        room.updateTopic(topic, it)
+    }
+
+    fun updateName(name: String): Completable = completableBuilder<Unit> {
+        room.updateName(name, it)
+    }
+
+    fun addRoomAlias(alias: String): Completable = completableBuilder<Unit> {
+        room.addRoomAlias(alias, it)
+    }
+
+    fun updateCanonicalAlias(alias: String): Completable = completableBuilder<Unit> {
+        room.updateCanonicalAlias(alias, it)
+    }
+
+    fun updateHistoryReadability(readability: RoomHistoryVisibility): Completable = completableBuilder<Unit> {
+        room.updateHistoryReadability(readability, it)
+    }
+
+    fun updateAvatar(avatarUri: Uri, fileName: String): Completable = completableBuilder<Unit> {
+        room.updateAvatar(avatarUri, fileName, it)
+    }
 }
 
 fun Room.rx(): RxRoom {

matrix-sdk-android/build.gradle

@@ -31,6 +31,11 @@ android {
         multiDexEnabled true
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
 
+        // The following argument makes the Android Test Orchestrator run its
+        // "pm clear" command after each test invocation. This command ensures
+        // that the app's state is completely cleared between tests.
+        testInstrumentationRunnerArguments clearPackageData: 'true'
+
         buildConfigField "String", "GIT_SDK_REVISION", "\"${gitRevision()}\""
         resValue "string", "git_sdk_revision", "\"${gitRevision()}\""
         resValue "string", "git_sdk_revision_unix_date", "\"${gitRevisionUnixDate()}\""
@@ -41,8 +46,11 @@ android {
         }
     }
 
-    buildTypes {
+    testOptions {
+        execution 'ANDROIDX_TEST_ORCHESTRATOR'
+    }
 
+    buildTypes {
         debug {
             // Set to true to log privacy or sensible data, such as token
             buildConfigField "boolean", "LOG_PRIVATE_DATA", project.property("vector.debugPrivateData")
@@ -114,7 +122,7 @@ dependencies {
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
 
     implementation "androidx.appcompat:appcompat:1.1.0"
-    implementation "androidx.core:core-ktx:1.1.0"
+    implementation "androidx.core:core-ktx:1.3.0"
 
     implementation "androidx.lifecycle:lifecycle-extensions:$lifecycle_version"
     implementation "androidx.lifecycle:lifecycle-common-java8:$lifecycle_version"
@@ -181,6 +189,7 @@ dependencies {
     // Plant Timber tree for test
     testImplementation 'net.lachlanmckee:timber-junit-rule:1.0.1'
 
+    kaptAndroidTest "com.google.dagger:dagger-compiler:$daggerVersion"
     androidTestImplementation 'androidx.test:core:1.2.0'
     androidTestImplementation 'androidx.test:runner:1.2.0'
     androidTestImplementation 'androidx.test:rules:1.2.0'
@@ -193,4 +202,6 @@ dependencies {
     androidTestImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
     // Plant Timber tree for test
     androidTestImplementation 'net.lachlanmckee:timber-junit-rule:1.0.1'
+
+    androidTestUtil 'androidx.test:orchestrator:1.2.0'
 }

matrix-sdk-android/proguard-rules.pro

@@ -64,3 +64,19 @@
 
 ### Webrtc
 -keep class org.webrtc.** { *; }
+
+### Serializable persisted classes
+# https://www.guardsquare.com/en/products/proguard/manual/examples#serializable
+-keepnames class * implements java.io.Serializable
+
+-keepclassmembers class * implements java.io.Serializable {
+    static final long serialVersionUID;
+    private static final java.io.ObjectStreamField[] serialPersistentFields;
+    !static !transient <fields>;
+    !private <fields>;
+    !private <methods>;
+    private void writeObject(java.io.ObjectOutputStream);
+    private void readObject(java.io.ObjectInputStream);
+    java.lang.Object writeReplace();
+    java.lang.Object readResolve();
+}

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/api/Matrix.kt

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.api
+
+import android.content.Context
+import androidx.lifecycle.ProcessLifecycleOwner
+import androidx.work.Configuration
+import androidx.work.WorkManager
+import com.zhuinden.monarchy.Monarchy
+import im.vector.matrix.android.BuildConfig
+import im.vector.matrix.android.api.auth.AuthenticationService
+import im.vector.matrix.android.common.DaggerTestMatrixComponent
+import im.vector.matrix.android.internal.SessionManager
+import im.vector.matrix.android.internal.crypto.attachments.ElementToDecrypt
+import im.vector.matrix.android.internal.crypto.attachments.MXEncryptedAttachments
+import im.vector.matrix.android.internal.network.UserAgentHolder
+import im.vector.matrix.android.internal.util.BackgroundDetectionObserver
+import org.matrix.olm.OlmManager
+import java.io.InputStream
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicBoolean
+import javax.inject.Inject
+
+/**
+ * This is the main entry point to the matrix sdk.
+ * To get the singleton instance, use getInstance static method.
+ */
+class Matrix private constructor(context: Context, matrixConfiguration: MatrixConfiguration)  {
+
+    @Inject internal lateinit var authenticationService: AuthenticationService
+    @Inject internal lateinit var userAgentHolder: UserAgentHolder
+    @Inject internal lateinit var backgroundDetectionObserver: BackgroundDetectionObserver
+    @Inject internal lateinit var olmManager: OlmManager
+    @Inject internal lateinit var sessionManager: SessionManager
+
+    init {
+        Monarchy.init(context)
+        DaggerTestMatrixComponent.factory().create(context, matrixConfiguration).inject(this)
+        if (context.applicationContext !is Configuration.Provider) {
+            WorkManager.initialize(context, Configuration.Builder().setExecutor(Executors.newCachedThreadPool()).build())
+        }
+        ProcessLifecycleOwner.get().lifecycle.addObserver(backgroundDetectionObserver)
+    }
+
+    fun getUserAgent() = userAgentHolder.userAgent
+
+    fun authenticationService(): AuthenticationService {
+        return authenticationService
+    }
+
+    companion object {
+
+        private lateinit var instance: Matrix
+        private val isInit = AtomicBoolean(false)
+
+        fun initialize(context: Context, matrixConfiguration: MatrixConfiguration) {
+            if (isInit.compareAndSet(false, true)) {
+                instance = Matrix(context.applicationContext, matrixConfiguration)
+            }
+        }
+
+        fun getInstance(context: Context): Matrix {
+            if (isInit.compareAndSet(false, true)) {
+                val appContext = context.applicationContext
+                if (appContext is MatrixConfiguration.Provider) {
+                    val matrixConfiguration = (appContext as MatrixConfiguration.Provider).providesMatrixConfiguration()
+                    instance = Matrix(appContext, matrixConfiguration)
+                } else {
+                    throw IllegalStateException("Matrix is not initialized properly." +
+                            " You should call Matrix.initialize or let your application implements MatrixConfiguration.Provider.")
+                }
+            }
+            return instance
+        }
+
+        fun getSdkVersion(): String {
+            return BuildConfig.VERSION_NAME + " (" + BuildConfig.GIT_SDK_REVISION + ")"
+        }
+
+        fun decryptStream(inputStream: InputStream?, elementToDecrypt: ElementToDecrypt): InputStream? {
+            return MXEncryptedAttachments.decryptAttachment(inputStream, elementToDecrypt)
+        }
+    }
+}

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/CommonTestHelper.kt

@@ -57,9 +57,10 @@ class CommonTestHelper(context: Context) {
 
     val matrix: Matrix
 
+    fun getTestInterceptor(session: Session): MockOkHttpInterceptor? = TestNetworkModule.interceptorForSession(session.sessionId) as? MockOkHttpInterceptor
+
     init {
         Matrix.initialize(context, MatrixConfiguration("TestFlavor"))
-
         matrix = Matrix.getInstance(context)
     }
 
@@ -116,6 +117,7 @@ class CommonTestHelper(context: Context) {
      * @param nbOfMessages the number of time the message will be sent
      */
     fun sendTextMessage(room: Room, message: String, nbOfMessages: Int): List<TimelineEvent> {
+        val timeline = room.createTimeline(null, TimelineSettings(10))
         val sentEvents = ArrayList<TimelineEvent>(nbOfMessages)
         val latch = CountDownLatch(1)
         val timelineListener = object : Timeline.Listener {
@@ -134,11 +136,12 @@ class CommonTestHelper(context: Context) {
 
                 if (newMessages.size == nbOfMessages) {
                     sentEvents.addAll(newMessages)
+                    // Remove listener now, if not at the next update sendEvents could change
+                    timeline.removeListener(this)
                     latch.countDown()
                 }
             }
         }
-        val timeline = room.createTimeline(null, TimelineSettings(10))
         timeline.start()
         timeline.addListener(timelineListener)
         for (i in 0 until nbOfMessages) {
@@ -146,11 +149,10 @@ class CommonTestHelper(context: Context) {
         }
         // Wait 3 second more per message
         await(latch, timeout = TestConstants.timeOutMillis + 3_000L * nbOfMessages)
-        timeline.removeListener(timelineListener)
         timeline.dispose()
 
         // Check that all events has been created
-        assertEquals(nbOfMessages.toLong(), sentEvents.size.toLong())
+        assertEquals("Message number do not match $sentEvents", nbOfMessages.toLong(), sentEvents.size.toLong())
 
         return sentEvents
     }

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/CryptoTestHelper.kt

@@ -17,8 +17,13 @@
 package im.vector.matrix.android.common
 
 import android.os.SystemClock
+import android.util.Log
 import androidx.lifecycle.Observer
 import im.vector.matrix.android.api.session.Session
+import im.vector.matrix.android.api.session.crypto.verification.IncomingSasVerificationTransaction
+import im.vector.matrix.android.api.session.crypto.verification.OutgoingSasVerificationTransaction
+import im.vector.matrix.android.api.session.crypto.verification.VerificationMethod
+import im.vector.matrix.android.api.session.crypto.verification.VerificationTxState
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.events.model.toContent
@@ -34,6 +39,7 @@ import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import im.vector.matrix.android.internal.crypto.keysbackup.model.MegolmBackupAuthData
 import im.vector.matrix.android.internal.crypto.keysbackup.model.MegolmBackupCreationInfo
+import im.vector.matrix.android.internal.crypto.model.rest.UserPasswordAuth
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
@@ -41,6 +47,8 @@ import kotlinx.coroutines.runBlocking
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import java.util.UUID
 import java.util.concurrent.CountDownLatch
 
 class CryptoTestHelper(private val mTestHelper: CommonTestHelper) {
@@ -274,4 +282,141 @@ class CryptoTestHelper(private val mTestHelper: CommonTestHelper) {
                 authData = createFakeMegolmBackupAuthData()
         )
     }
+
+    fun createDM(alice: Session, bob: Session): String {
+        val roomId = mTestHelper.doSync<String> {
+            alice.createRoom(
+                    CreateRoomParams(invitedUserIds = listOf(bob.myUserId))
+                            .setDirectMessage()
+                            .enableEncryptionIfInvitedUsersSupportIt(),
+                    it
+            )
+        }
+
+        mTestHelper.waitWithLatch { latch ->
+            val bobRoomSummariesLive = runBlocking(Dispatchers.Main) {
+                bob.getRoomSummariesLive(roomSummaryQueryParams { })
+            }
+
+            val newRoomObserver = object : Observer<List<RoomSummary>> {
+                override fun onChanged(t: List<RoomSummary>?) {
+                    val indexOfFirst = t?.indexOfFirst { it.roomId == roomId } ?: -1
+                    if (indexOfFirst != -1) {
+                        latch.countDown()
+                        bobRoomSummariesLive.removeObserver(this)
+                    }
+                }
+            }
+
+            GlobalScope.launch(Dispatchers.Main) {
+                bobRoomSummariesLive.observeForever(newRoomObserver)
+            }
+        }
+
+        mTestHelper.waitWithLatch { latch ->
+            val bobRoomSummariesLive = runBlocking(Dispatchers.Main) {
+                bob.getRoomSummariesLive(roomSummaryQueryParams { })
+            }
+
+            val newRoomObserver = object : Observer<List<RoomSummary>> {
+                override fun onChanged(t: List<RoomSummary>?) {
+                    if (bob.getRoom(roomId)
+                                    ?.getRoomMember(bob.myUserId)
+                                    ?.membership == Membership.JOIN) {
+                        latch.countDown()
+                        bobRoomSummariesLive.removeObserver(this)
+                    }
+                }
+            }
+
+            GlobalScope.launch(Dispatchers.Main) {
+                bobRoomSummariesLive.observeForever(newRoomObserver)
+            }
+
+            mTestHelper.doSync<Unit> { bob.joinRoom(roomId, callback = it) }
+        }
+
+        return roomId
+    }
+
+    fun initializeCrossSigning(session: Session) {
+        mTestHelper.doSync<Unit> {
+            session.cryptoService().crossSigningService()
+                    .initializeCrossSigning(UserPasswordAuth(
+                            user = session.myUserId,
+                            password = TestConstants.PASSWORD
+                    ), it)
+        }
+    }
+
+    fun verifySASCrossSign(alice: Session, bob: Session, roomId: String) {
+        assertTrue(alice.cryptoService().crossSigningService().canCrossSign())
+        assertTrue(bob.cryptoService().crossSigningService().canCrossSign())
+
+        val requestID = UUID.randomUUID().toString()
+        val aliceVerificationService = alice.cryptoService().verificationService()
+        val bobVerificationService = bob.cryptoService().verificationService()
+
+        aliceVerificationService.beginKeyVerificationInDMs(
+                VerificationMethod.SAS,
+                requestID,
+                roomId,
+                bob.myUserId,
+                bob.sessionParams.credentials.deviceId!!,
+                null)
+
+        // we should reach SHOW SAS on both
+        var alicePovTx: OutgoingSasVerificationTransaction? = null
+        var bobPovTx: IncomingSasVerificationTransaction? = null
+
+        // wait for alice to get the ready
+        mTestHelper.waitWithLatch {
+            mTestHelper.retryPeriodicallyWithLatch(it) {
+                bobPovTx = bobVerificationService.getExistingTransaction(alice.myUserId, requestID) as? IncomingSasVerificationTransaction
+                Log.v("TEST", "== bobPovTx is ${alicePovTx?.uxState}")
+                if (bobPovTx?.state == VerificationTxState.OnStarted) {
+                    bobPovTx?.performAccept()
+                    true
+                } else {
+                    false
+                }
+            }
+        }
+
+        mTestHelper.waitWithLatch {
+            mTestHelper.retryPeriodicallyWithLatch(it) {
+                alicePovTx = aliceVerificationService.getExistingTransaction(bob.myUserId, requestID) as? OutgoingSasVerificationTransaction
+                Log.v("TEST", "== alicePovTx is ${alicePovTx?.uxState}")
+                alicePovTx?.state == VerificationTxState.ShortCodeReady
+            }
+        }
+        // wait for alice to get the ready
+        mTestHelper.waitWithLatch {
+            mTestHelper.retryPeriodicallyWithLatch(it) {
+                bobPovTx = bobVerificationService.getExistingTransaction(alice.myUserId, requestID) as? IncomingSasVerificationTransaction
+                Log.v("TEST", "== bobPovTx is ${alicePovTx?.uxState}")
+                if (bobPovTx?.state == VerificationTxState.OnStarted) {
+                    bobPovTx?.performAccept()
+                }
+                bobPovTx?.state == VerificationTxState.ShortCodeReady
+            }
+        }
+
+        assertEquals("SAS code do not match", alicePovTx!!.getDecimalCodeRepresentation(), bobPovTx!!.getDecimalCodeRepresentation())
+
+        bobPovTx!!.userHasVerifiedShortCode()
+        alicePovTx!!.userHasVerifiedShortCode()
+
+        mTestHelper.waitWithLatch {
+            mTestHelper.retryPeriodicallyWithLatch(it) {
+                alice.cryptoService().crossSigningService().isUserTrusted(bob.myUserId)
+            }
+        }
+
+        mTestHelper.waitWithLatch {
+            mTestHelper.retryPeriodicallyWithLatch(it) {
+                alice.cryptoService().crossSigningService().isUserTrusted(bob.myUserId)
+            }
+        }
+    }
 }

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/MockOkHttpInterceptor.kt

@@ -15,6 +15,7 @@
  */
 package im.vector.matrix.android.common
 
+import im.vector.matrix.android.internal.session.TestInterceptor
 import okhttp3.Interceptor
 import okhttp3.Protocol
 import okhttp3.Request
@@ -37,7 +38,7 @@ import javax.net.ssl.HttpsURLConnection
  *      AutoDiscovery().findClientConfig("matrix.org", <callback>)
  * </code>
  */
-class MockOkHttpInterceptor : Interceptor {
+class MockOkHttpInterceptor : TestInterceptor {
 
     private var rules: ArrayList<Rule> = ArrayList()
 
@@ -45,6 +46,12 @@ class MockOkHttpInterceptor : Interceptor {
         rules.add(rule)
     }
 
+    fun clearRules() {
+        rules.clear()
+    }
+
+    override var sessionId: String? = null
+
     override fun intercept(chain: Interceptor.Chain): Response {
         val originalRequest = chain.request()
 

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/TestMatrixComponent.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.common
+
+import android.content.Context
+import dagger.BindsInstance
+import dagger.Component
+import im.vector.matrix.android.api.MatrixConfiguration
+import im.vector.matrix.android.internal.auth.AuthModule
+import im.vector.matrix.android.internal.di.MatrixComponent
+import im.vector.matrix.android.internal.di.MatrixModule
+import im.vector.matrix.android.internal.di.MatrixScope
+import im.vector.matrix.android.internal.di.NetworkModule
+
+@Component(modules = [TestModule::class, MatrixModule::class, NetworkModule::class, AuthModule::class, TestNetworkModule::class])
+@MatrixScope
+internal interface TestMatrixComponent : MatrixComponent {
+
+    @Component.Factory
+    interface Factory {
+        fun create(@BindsInstance context: Context,
+                   @BindsInstance matrixConfiguration: MatrixConfiguration): TestMatrixComponent
+    }
+}

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/TestModule.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.common
+
+import dagger.Binds
+import dagger.Module
+import im.vector.matrix.android.internal.di.MatrixComponent
+
+@Module
+internal abstract class TestModule {
+    @Binds
+    abstract fun providesMatrixComponent(testMatrixComponent: TestMatrixComponent): MatrixComponent
+}

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/TestNetworkModule.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.common
+
+import dagger.Module
+import dagger.Provides
+import im.vector.matrix.android.internal.session.MockHttpInterceptor
+import im.vector.matrix.android.internal.session.TestInterceptor
+
+@Module
+internal object TestNetworkModule {
+
+    val interceptors = ArrayList<TestInterceptor>()
+
+    fun interceptorForSession(sessionId: String): TestInterceptor? = interceptors.firstOrNull { it.sessionId == sessionId }
+
+    @Provides
+    @JvmStatic
+    @MockHttpInterceptor
+    fun providesTestInterceptor(): TestInterceptor? {
+        return MockOkHttpInterceptor().also {
+            interceptors.add(it)
+        }
+    }
+}

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/internal/crypto/CryptoStoreTest.kt

@@ -22,10 +22,8 @@ import im.vector.matrix.android.internal.crypto.model.OlmSessionWrapper
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import io.realm.Realm
 import org.junit.Assert.assertEquals
-import org.junit.Assert.assertFalse
 import org.junit.Assert.assertNotEquals
 import org.junit.Assert.assertNull
-import org.junit.Assert.assertTrue
 import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -45,22 +43,22 @@ class CryptoStoreTest : InstrumentedTest {
         Realm.init(context())
     }
 
-    @Test
-    fun test_metadata_realm_ok() {
-        val cryptoStore: IMXCryptoStore = cryptoStoreHelper.createStore()
-
-        assertFalse(cryptoStore.hasData())
-
-        cryptoStore.open()
-
-        assertEquals("deviceId_sample", cryptoStore.getDeviceId())
-
-        assertTrue(cryptoStore.hasData())
-
-        // Cleanup
-        cryptoStore.close()
-        cryptoStore.deleteStore()
-    }
+//    @Test
+//    fun test_metadata_realm_ok() {
+//        val cryptoStore: IMXCryptoStore = cryptoStoreHelper.createStore()
+//
+//        assertFalse(cryptoStore.hasData())
+//
+//        cryptoStore.open()
+//
+//        assertEquals("deviceId_sample", cryptoStore.getDeviceId())
+//
+//        assertTrue(cryptoStore.hasData())
+//
+//        // Cleanup
+//        cryptoStore.close()
+//        cryptoStore.deleteStore()
+//    }
 
     @Test
     fun test_lastSessionUsed() {

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/internal/crypto/gossiping/KeyShareTests.kt

@@ -29,6 +29,7 @@ import im.vector.matrix.android.api.session.events.model.toModel
 import im.vector.matrix.android.api.session.room.model.RoomDirectoryVisibility
 import im.vector.matrix.android.api.session.room.model.create.CreateRoomParams
 import im.vector.matrix.android.common.CommonTestHelper
+import im.vector.matrix.android.common.CryptoTestHelper
 import im.vector.matrix.android.common.SessionTestParams
 import im.vector.matrix.android.common.TestConstants
 import im.vector.matrix.android.internal.crypto.GossipingRequestState
@@ -56,6 +57,7 @@ import java.util.concurrent.CountDownLatch
 class KeyShareTests : InstrumentedTest {
 
     private val mTestHelper = CommonTestHelper(context())
+    private val mCryptoTestHelper = CryptoTestHelper(mTestHelper)
 
     @Test
     fun test_DoNotSelfShareIfNotTrusted() {
@@ -234,6 +236,7 @@ class KeyShareTests : InstrumentedTest {
                     }
                     if (tx.state == VerificationTxState.ShortCodeReady) {
                         session1ShortCode = tx.getDecimalCodeRepresentation()
+                        Thread.sleep(500)
                         tx.userHasVerifiedShortCode()
                     }
                 }
@@ -246,6 +249,7 @@ class KeyShareTests : InstrumentedTest {
                 if (tx is SasVerificationTransaction) {
                     if (tx.state == VerificationTxState.ShortCodeReady) {
                         session2ShortCode = tx.getDecimalCodeRepresentation()
+                        Thread.sleep(500)
                         tx.userHasVerifiedShortCode()
                     }
                 }
@@ -285,5 +289,8 @@ class KeyShareTests : InstrumentedTest {
                 keysBackupService.getKeyBackupRecoveryKeyInfo()?.recoveryKey == creationInfo.recoveryKey
             }
         }
+
+        mTestHelper.signOutAndClose(aliceSession1)
+        mTestHelper.signOutAndClose(aliceSession2)
     }
 }

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/internal/crypto/gossiping/WithHeldTests.kt

@@ -0,0 +1,245 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.gossiping
+
+import android.util.Log
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import im.vector.matrix.android.InstrumentedTest
+import im.vector.matrix.android.api.NoOpMatrixCallback
+import im.vector.matrix.android.api.extensions.tryThis
+import im.vector.matrix.android.api.session.crypto.MXCryptoError
+import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.toModel
+import im.vector.matrix.android.common.CommonTestHelper
+import im.vector.matrix.android.common.CryptoTestHelper
+import im.vector.matrix.android.common.MockOkHttpInterceptor
+import im.vector.matrix.android.common.SessionTestParams
+import im.vector.matrix.android.common.TestConstants
+import im.vector.matrix.android.internal.crypto.model.event.EncryptedEventContent
+import im.vector.matrix.android.internal.crypto.model.event.WithHeldCode
+import org.junit.Assert
+import org.junit.FixMethodOrder
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.MethodSorters
+
+@RunWith(AndroidJUnit4::class)
+@FixMethodOrder(MethodSorters.JVM)
+class WithHeldTests : InstrumentedTest {
+
+    private val mTestHelper = CommonTestHelper(context())
+    private val mCryptoTestHelper = CryptoTestHelper(mTestHelper)
+
+    @Test
+    fun test_WithHeldUnverifiedReason() {
+        // =============================
+        // ARRANGE
+        // =============================
+
+        val aliceSession = mTestHelper.createAccount(TestConstants.USER_ALICE, SessionTestParams(true))
+        val bobSession = mTestHelper.createAccount(TestConstants.USER_ALICE, SessionTestParams(true))
+
+        // Initialize cross signing on both
+        mCryptoTestHelper.initializeCrossSigning(aliceSession)
+        mCryptoTestHelper.initializeCrossSigning(bobSession)
+
+        val roomId = mCryptoTestHelper.createDM(aliceSession, bobSession)
+        mCryptoTestHelper.verifySASCrossSign(aliceSession, bobSession, roomId)
+
+        val roomAlicePOV = aliceSession.getRoom(roomId)!!
+
+        val bobUnverifiedSession = mTestHelper.logIntoAccount(bobSession.myUserId, SessionTestParams(true))
+
+        // =============================
+        // ACT
+        // =============================
+
+        // Alice decide to not send to unverified sessions
+        aliceSession.cryptoService().setGlobalBlacklistUnverifiedDevices(true)
+
+        val timelineEvent = mTestHelper.sendTextMessage(roomAlicePOV, "Hello Bob", 1).first()
+
+        // await for bob unverified session to get the message
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                bobUnverifiedSession.getRoom(roomId)?.getTimeLineEvent(timelineEvent.eventId) != null
+            }
+        }
+
+        val eventBobPOV = bobUnverifiedSession.getRoom(roomId)?.getTimeLineEvent(timelineEvent.eventId)!!
+
+        // =============================
+        // ASSERT
+        // =============================
+
+        // Bob should not be able to decrypt because the keys is withheld
+        try {
+            // .. might need to wait a bit for stability?
+            bobUnverifiedSession.cryptoService().decryptEvent(eventBobPOV.root, "")
+            Assert.fail("This session should not be able to decrypt")
+        } catch (failure: Throwable) {
+            val type = (failure as MXCryptoError.Base).errorType
+            val technicalMessage = failure.technicalMessage
+            Assert.assertEquals("Error should be withheld", MXCryptoError.ErrorType.KEYS_WITHHELD, type)
+            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNVERIFIED.value, technicalMessage)
+        }
+
+        // enable back sending to unverified
+        aliceSession.cryptoService().setGlobalBlacklistUnverifiedDevices(false)
+
+        val secondEvent = mTestHelper.sendTextMessage(roomAlicePOV, "Verify your device!!", 1).first()
+
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                val ev = bobUnverifiedSession.getRoom(roomId)?.getTimeLineEvent(secondEvent.eventId)
+                // wait until it's decrypted
+                ev?.root?.getClearType() == EventType.MESSAGE
+            }
+        }
+
+        // Previous message should still be undecryptable (partially withheld session)
+        try {
+            // .. might need to wait a bit for stability?
+            bobUnverifiedSession.cryptoService().decryptEvent(eventBobPOV.root, "")
+            Assert.fail("This session should not be able to decrypt")
+        } catch (failure: Throwable) {
+            val type = (failure as MXCryptoError.Base).errorType
+            val technicalMessage = failure.technicalMessage
+            Assert.assertEquals("Error should be withheld", MXCryptoError.ErrorType.KEYS_WITHHELD, type)
+            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNVERIFIED.value, technicalMessage)
+        }
+
+        mTestHelper.signOutAndClose(aliceSession)
+        mTestHelper.signOutAndClose(bobSession)
+        mTestHelper.signOutAndClose(bobUnverifiedSession)
+    }
+
+    @Test
+    fun  test_WithHeldNoOlm() {
+        val testData = mCryptoTestHelper.doE2ETestWithAliceAndBobInARoom()
+        val aliceSession = testData.firstSession
+        val bobSession = testData.secondSession!!
+        val aliceInterceptor = mTestHelper.getTestInterceptor(aliceSession)
+
+        // Simulate no OTK
+        aliceInterceptor!!.addRule(MockOkHttpInterceptor.SimpleRule(
+                "/keys/claim",
+                200,
+                """
+                   { "one_time_keys" : {} } 
+                """
+        ))
+        Log.d("#TEST", "Recovery :${aliceSession.sessionParams.credentials.accessToken}")
+
+        val roomAlicePov = aliceSession.getRoom(testData.roomId)!!
+
+        val eventId = mTestHelper.sendTextMessage(roomAlicePov, "first message", 1).first().eventId
+
+        // await for bob session to get the message
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                bobSession.getRoom(testData.roomId)?.getTimeLineEvent(eventId) != null
+            }
+        }
+
+        // Previous message should still be undecryptable (partially withheld session)
+        val eventBobPOV = bobSession.getRoom(testData.roomId)?.getTimeLineEvent(eventId)
+        try {
+            // .. might need to wait a bit for stability?
+            bobSession.cryptoService().decryptEvent(eventBobPOV!!.root, "")
+            Assert.fail("This session should not be able to decrypt")
+        } catch (failure: Throwable) {
+            val type = (failure as MXCryptoError.Base).errorType
+            val technicalMessage = failure.technicalMessage
+            Assert.assertEquals("Error should be withheld", MXCryptoError.ErrorType.KEYS_WITHHELD, type)
+            Assert.assertEquals("Cause should be unverified", WithHeldCode.NO_OLM.value, technicalMessage)
+        }
+
+        // Ensure that alice has marked the session to be shared with bob
+        val sessionId = eventBobPOV!!.root.content.toModel<EncryptedEventContent>()!!.sessionId!!
+        val chainIndex = aliceSession.cryptoService().getSharedWithInfo(testData.roomId, sessionId).getObject(bobSession.myUserId, bobSession.sessionParams.credentials.deviceId)
+
+        Assert.assertEquals("Alice should have marked bob's device for this session", 0, chainIndex)
+        // Add a new device for bob
+
+        aliceInterceptor.clearRules()
+        val bobSecondSession = mTestHelper.logIntoAccount(bobSession.myUserId, SessionTestParams(withInitialSync = true))
+        // send a second message
+        val secondMessageId = mTestHelper.sendTextMessage(roomAlicePov, "second message", 1).first().eventId
+
+        // Check that the
+        // await for bob SecondSession session to get the message
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                bobSecondSession.getRoom(testData.roomId)?.getTimeLineEvent(secondMessageId) != null
+            }
+        }
+
+        val chainIndex2 = aliceSession.cryptoService().getSharedWithInfo(testData.roomId, sessionId).getObject(bobSecondSession.myUserId, bobSecondSession.sessionParams.credentials.deviceId)
+
+        Assert.assertEquals("Alice should have marked bob's device for this session", 1, chainIndex2)
+
+        aliceInterceptor.clearRules()
+        testData.cleanUp(mTestHelper)
+        mTestHelper.signOutAndClose(bobSecondSession)
+    }
+
+    @Test
+    fun test_WithHeldKeyRequest() {
+        val testData = mCryptoTestHelper.doE2ETestWithAliceAndBobInARoom()
+        val aliceSession = testData.firstSession
+        val bobSession = testData.secondSession!!
+
+        val roomAlicePov = aliceSession.getRoom(testData.roomId)!!
+
+        val eventId = mTestHelper.sendTextMessage(roomAlicePov, "first message", 1).first().eventId
+
+        mTestHelper.signOutAndClose(bobSession)
+
+        // Create a new session for bob
+
+        val bobSecondSession = mTestHelper.logIntoAccount(bobSession.myUserId, SessionTestParams(true))
+        // initialize to force request keys if missing
+        mCryptoTestHelper.initializeCrossSigning(bobSecondSession)
+
+        // Trust bob second device from Alice POV
+        aliceSession.cryptoService().crossSigningService().trustDevice(bobSecondSession.sessionParams.deviceId!!, NoOpMatrixCallback())
+        bobSecondSession.cryptoService().crossSigningService().trustDevice(aliceSession.sessionParams.deviceId!!, NoOpMatrixCallback())
+
+        var sessionId: String? = null
+        // Check that the
+        // await for bob SecondSession session to get the message
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                val timeLineEvent = bobSecondSession.getRoom(testData.roomId)?.getTimeLineEvent(eventId)?.also {
+                    // try to decrypt and force key request
+                    tryThis { bobSecondSession.cryptoService().decryptEvent(it.root, "") }
+                }
+                sessionId = timeLineEvent?.root?.content?.toModel<EncryptedEventContent>()?.sessionId
+                timeLineEvent != null
+            }
+        }
+
+        // Check that bob second session requested the key
+        mTestHelper.waitWithLatch { latch ->
+            mTestHelper.retryPeriodicallyWithLatch(latch) {
+                val wc = bobSecondSession.cryptoService().getWithHeldMegolmSession(roomAlicePov.roomId, sessionId!!)
+                wc?.code == WithHeldCode.UNAUTHORISED
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/AndroidManifest.xml

@@ -1,5 +1,4 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:tools="http://schemas.android.com/tools"
     package="im.vector.matrix.android">
 
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
@@ -8,13 +7,20 @@
 
     <application android:networkSecurityConfig="@xml/network_security_config">
 
+        <!--
+         The SDK offers a secured File provider to access downloaded files.
+         Access to these file will be given via the FileService, with a temporary
+         read access permission
+        -->
         <provider
-            android:name="androidx.work.impl.WorkManagerInitializer"
-            android:authorities="${applicationId}.workmanager-init"
+            android:name="im.vector.matrix.android.api.session.file.MatrixSDKFileProvider"
+            android:authorities="${applicationId}.mx-sdk.fileprovider"
             android:exported="false"
-            tools:node="remove" />
-
+            android:grantUriPermissions="true">
+            <meta-data
+                android:name="android.support.FILE_PROVIDER_PATHS"
+                android:resource="@xml/sdk_provider_paths" />
+        </provider>
     </application>
 
-
 </manifest>

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/auth/AuthenticationService.kt

@@ -89,6 +89,7 @@ interface AuthenticationService {
      * Perform a wellknown request, using the domain from the matrixId
      */
     fun getWellKnownData(matrixId: String,
+                         homeServerConnectionConfig: HomeServerConnectionConfig?,
                          callback: MatrixCallback<WellknownResult>): Cancelable
 
     /**

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/failure/Failure.kt

@@ -18,6 +18,7 @@ package im.vector.matrix.android.api.failure
 
 import im.vector.matrix.android.api.session.crypto.MXCryptoError
 import im.vector.matrix.android.internal.auth.registration.RegistrationFlowResponse
+import im.vector.matrix.android.internal.network.ssl.Fingerprint
 import java.io.IOException
 
 /**
@@ -32,9 +33,11 @@ import java.io.IOException
 sealed class Failure(cause: Throwable? = null) : Throwable(cause = cause) {
     data class Unknown(val throwable: Throwable? = null) : Failure(throwable)
     data class Cancelled(val throwable: Throwable? = null) : Failure(throwable)
+    data class UnrecognizedCertificateFailure(val url: String, val fingerprint: Fingerprint) : Failure()
     data class NetworkConnection(val ioException: IOException? = null) : Failure(ioException)
     data class ServerError(val error: MatrixError, val httpCode: Int) : Failure(RuntimeException(error.toString()))
     object SuccessError : Failure(RuntimeException(RuntimeException("SuccessResult is false")))
+
     // When server send an error, but it cannot be interpreted as a MatrixError
     data class OtherServerError(val errorBody: String, val httpCode: Int) : Failure(RuntimeException("HTTP $httpCode: $errorBody"))
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/failure/GlobalError.kt

@@ -16,8 +16,11 @@
 
 package im.vector.matrix.android.api.failure
 
+import im.vector.matrix.android.internal.network.ssl.Fingerprint
+
 // This class will be sent to the bus
 sealed class GlobalError {
     data class InvalidToken(val softLogout: Boolean) : GlobalError()
     data class ConsentNotGivenError(val consentUri: String) : GlobalError()
+    data class CertificateError(val fingerprint: Fingerprint) : GlobalError()
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt

@@ -28,6 +28,7 @@ import im.vector.matrix.android.api.session.call.CallSignalingService
 import im.vector.matrix.android.api.session.content.ContentUploadStateTracker
 import im.vector.matrix.android.api.session.content.ContentUrlResolver
 import im.vector.matrix.android.api.session.crypto.CryptoService
+import im.vector.matrix.android.api.session.file.ContentDownloadStateTracker
 import im.vector.matrix.android.api.session.file.FileService
 import im.vector.matrix.android.api.session.group.GroupService
 import im.vector.matrix.android.api.session.homeserver.HomeServerCapabilitiesService
@@ -59,7 +60,6 @@ interface Session :
         CacheService,
         SignOutService,
         FilterService,
-        FileService,
         TermsService,
         ProfileService,
         PushRuleService,
@@ -127,6 +127,12 @@ interface Session :
      */
     fun getSyncStateLive(): LiveData<SyncState>
 
+    /**
+     * This method returns the current sync state.
+     * @return the current [SyncState].
+     */
+    fun getSyncState(): SyncState
+
     /**
      * This methods return true if an initial sync has been processed
      */
@@ -152,6 +158,11 @@ interface Session :
      */
     fun typingUsersTracker(): TypingUsersTracker
 
+    /**
+     * Returns the ContentDownloadStateTracker associated with the session
+     */
+    fun contentDownloadProgressTracker(): ContentDownloadStateTracker
+
     /**
      * Returns the cryptoService associated with the session
      */
@@ -177,6 +188,11 @@ interface Session :
      */
     fun callSignalingService(): CallSignalingService
 
+    /**
+     * Returns the file download service associated with the session
+     */
+    fun fileService(): FileService
+
     /**
      * Add a listener to the session.
      * @param listener the listener to add.

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/crypto/CryptoService.kt

@@ -36,6 +36,7 @@ import im.vector.matrix.android.internal.crypto.model.ImportRoomKeysResult
 import im.vector.matrix.android.internal.crypto.model.MXDeviceInfo
 import im.vector.matrix.android.internal.crypto.model.MXEncryptEventContentResult
 import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
 import im.vector.matrix.android.internal.crypto.model.rest.DeviceInfo
 import im.vector.matrix.android.internal.crypto.model.rest.DevicesListResponse
 import im.vector.matrix.android.internal.crypto.model.rest.RoomKeyRequestBody
@@ -145,4 +146,8 @@ interface CryptoService {
     fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
 
     fun getGossipingEventsTrail(): List<Event>
+
+    // For testing shared session
+    fun getSharedWithInfo(roomId: String?, sessionId: String) : MXUsersDevicesMap<Int>
+    fun getWithHeldMegolmSession(roomId: String, sessionId: String) : RoomKeyWithHeldContent?
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/crypto/MXCryptoError.kt

@@ -59,7 +59,8 @@ sealed class MXCryptoError : Throwable() {
         MISSING_PROPERTY,
         OLM,
         UNKNOWN_DEVICES,
-        UNKNOWN_MESSAGE_INDEX
+        UNKNOWN_MESSAGE_INDEX,
+        KEYS_WITHHELD
     }
 
     companion object {

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/Event.kt

@@ -82,6 +82,9 @@ data class Event(
     @Transient
     var mCryptoError: MXCryptoError.ErrorType? = null
 
+    @Transient
+    var mCryptoErrorReason: String? = null
+
     @Transient
     var sendState: SendState = SendState.UNKNOWN
 
@@ -182,6 +185,7 @@ data class Event(
         if (redacts != other.redacts) return false
         if (mxDecryptionResult != other.mxDecryptionResult) return false
         if (mCryptoError != other.mCryptoError) return false
+        if (mCryptoErrorReason != other.mCryptoErrorReason) return false
         if (sendState != other.sendState) return false
 
         return true
@@ -200,6 +204,7 @@ data class Event(
         result = 31 * result + (redacts?.hashCode() ?: 0)
         result = 31 * result + (mxDecryptionResult?.hashCode() ?: 0)
         result = 31 * result + (mCryptoError?.hashCode() ?: 0)
+        result = 31 * result + (mCryptoErrorReason?.hashCode() ?: 0)
         result = 31 * result + sendState.hashCode()
         return result
     }
@@ -230,3 +235,11 @@ fun Event.isVideoMessage(): Boolean {
         else                      -> false
     }
 }
+
+fun Event.isFileMessage(): Boolean {
+    return getClearType() == EventType.MESSAGE
+            && when (getClearContent()?.toModel<MessageContent>()?.msgType) {
+        MessageType.MSGTYPE_FILE -> true
+        else                      -> false
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt

@@ -66,6 +66,7 @@ object EventType {
     // Key share events
     const val ROOM_KEY_REQUEST = "m.room_key_request"
     const val FORWARDED_ROOM_KEY = "m.forwarded_room_key"
+    const val ROOM_KEY_WITHHELD = "org.matrix.room_key.withheld"
 
     const val REQUEST_SECRET = "m.secret.request"
     const val SEND_SECRET = "m.secret.send"

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/file/ContentDownloadStateTracker.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.api.session.file
+
+interface ContentDownloadStateTracker {
+    fun track(key: String, updateListener: UpdateListener)
+    fun unTrack(key: String, updateListener: UpdateListener)
+    fun clear()
+
+    sealed class State {
+        object Idle : State()
+        data class Downloading(val current: Long, val total: Long, val indeterminate: Boolean) : State()
+        object Decrypting : State()
+        object Success : State()
+        data class Failure(val errorCode: Int) : State()
+    }
+
+    interface UpdateListener {
+        fun onDownloadStateUpdate(state: State)
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/file/FileService.kt

@@ -16,6 +16,7 @@
 
 package im.vector.matrix.android.api.session.file
 
+import android.net.Uri
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.util.Cancelable
 import im.vector.matrix.android.internal.crypto.attachments.ElementToDecrypt
@@ -31,26 +32,58 @@ interface FileService {
          * Download file in external storage
          */
         TO_EXPORT,
+
         /**
          * Download file in cache
          */
         FOR_INTERNAL_USE,
+
         /**
          * Download file in file provider path
          */
         FOR_EXTERNAL_SHARE
     }
 
+    enum class FileState {
+        IN_CACHE,
+        DOWNLOADING,
+        UNKNOWN
+    }
+
     /**
      * Download a file.
-     * Result will be a decrypted file, stored in the cache folder. id parameter will be used to create a sub folder to avoid name collision.
-     * You can pass the eventId
+     * Result will be a decrypted file, stored in the cache folder. url parameter will be used to create unique filename to avoid name collision.
      */
     fun downloadFile(
             downloadMode: DownloadMode,
             id: String,
             fileName: String,
+            mimeType: String?,
             url: String?,
             elementToDecrypt: ElementToDecrypt?,
             callback: MatrixCallback<File>): Cancelable
+
+    fun isFileInCache(mxcUrl: String, mimeType: String?): Boolean
+
+    /**
+     * Use this URI and pass it to intent using flag Intent.FLAG_GRANT_READ_URI_PERMISSION
+     * (if not other app won't be able to access it)
+     */
+    fun getTemporarySharableURI(mxcUrl: String, mimeType: String?): Uri?
+
+    /**
+     * Get information on the given file.
+     * Mimetype should be the same one as passed to downloadFile (limitation for now)
+     */
+    fun fileState(mxcUrl: String, mimeType: String?): FileState
+
+    /**
+     * Clears all the files downloaded by the service
+     */
+    fun clearCache()
+
+    /**
+     * Get size of cached files
+     */
+    fun getCacheSize(): Int
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/file/MatrixSDKFileProvider.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.api.session.file
+
+import android.net.Uri
+import androidx.core.content.FileProvider
+
+/**
+ * We have to declare our own file provider to avoid collision with apps using the sdk
+ * and having their own
+ */
+class MatrixSDKFileProvider : FileProvider() {
+    override fun getType(uri: Uri): String? {
+        return super.getType(uri) ?: "plain/text"
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/group/Group.kt

@@ -16,9 +16,20 @@
 
 package im.vector.matrix.android.api.session.group
 
+import im.vector.matrix.android.api.MatrixCallback
+import im.vector.matrix.android.api.util.Cancelable
+
 /**
  * This interface defines methods to interact within a group.
  */
 interface Group {
     val groupId: String
+
+    /**
+     * This methods allows you to refresh data about this group. It will be reflected on the GroupSummary.
+     * The SDK also takes care of refreshing group data every hour.
+     * @param callback : the matrix callback to be notified of success or failure
+     * @return a Cancelable to be able to cancel requests.
+     */
+    fun fetchGroupData(callback: MatrixCallback<Unit>): Cancelable
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/profile/ProfileService.kt

@@ -17,6 +17,7 @@
 
 package im.vector.matrix.android.api.session.profile
 
+import android.net.Uri
 import androidx.lifecycle.LiveData
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.session.identity.ThreePid
@@ -48,6 +49,14 @@ interface ProfileService {
      */
     fun setDisplayName(userId: String, newDisplayName: String, matrixCallback: MatrixCallback<Unit>): Cancelable
 
+    /**
+     * Update the avatar for this user
+     * @param userId the userId to update the avatar of
+     * @param newAvatarUri the new avatar uri of the user
+     * @param fileName the fileName of selected image
+     */
+    fun updateAvatar(userId: String, newAvatarUri: Uri, fileName: String, matrixCallback: MatrixCallback<Unit>): Cancelable
+
     /**
      * Return the current avatarUrl for this user.
      * @param userId the userId param to look for

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt

@@ -19,6 +19,7 @@ package im.vector.matrix.android.api.session.room.model
 import im.vector.matrix.android.api.crypto.RoomEncryptionTrustLevel
 import im.vector.matrix.android.api.session.room.model.tag.RoomTag
 import im.vector.matrix.android.api.session.room.send.UserDraft
+import im.vector.matrix.android.api.session.room.sender.SenderInfo
 import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
 
 /**
@@ -27,7 +28,9 @@ import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
  */
 data class RoomSummary constructor(
         val roomId: String,
+        // Computed display name
         val displayName: String = "",
+        val name: String = "",
         val topic: String = "",
         val avatarUrl: String = "",
         val canonicalAlias: String? = null,
@@ -47,6 +50,7 @@ data class RoomSummary constructor(
         val userDrafts: List<UserDraft> = emptyList(),
         val isEncrypted: Boolean,
         val encryptionEventTs: Long?,
+        val typingUsers: List<SenderInfo>,
         val inviterId: String? = null,
         val breadcrumbsIndex: Int = NOT_IN_BREADCRUMBS,
         val roomEncryptionTrustLevel: RoomEncryptionTrustLevel? = null

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageAudioContent.kt

@@ -51,4 +51,8 @@ data class MessageAudioContent(
          * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
          */
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
-) : MessageWithAttachmentContent
+) : MessageWithAttachmentContent {
+
+        override val mimeType: String?
+                get() =  encryptedFileInfo?.mimetype ?: audioInfo?.mimeType
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageFileContent.kt

@@ -16,7 +16,7 @@
 
 package im.vector.matrix.android.api.session.room.model.message
 
-import android.content.ClipDescription
+import android.webkit.MimeTypeMap
 import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
 import im.vector.matrix.android.api.session.events.model.Content
@@ -59,12 +59,12 @@ data class MessageFileContent(
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
 ) : MessageWithAttachmentContent {
 
-    fun getMimeType(): String {
-        // Mimetype default to plain text, should not be used
-        return encryptedFileInfo?.mimetype
+    override val mimeType: String?
+        get() = encryptedFileInfo?.mimetype
                 ?: info?.mimeType
-                ?: ClipDescription.MIMETYPE_TEXT_PLAIN
-    }
+                ?: MimeTypeMap.getFileExtensionFromUrl(filename ?: body)?.let { extension ->
+                    MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension)
+                }
 
     fun getFileName(): String {
         return filename ?: body

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageImageContent.kt

@@ -52,4 +52,7 @@ data class MessageImageContent(
          * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
          */
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
-) : MessageImageInfoContent
+) : MessageImageInfoContent {
+        override val mimeType: String?
+                get() = encryptedFileInfo?.mimetype ?: info?.mimeType ?: "image/*"
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageStickerContent.kt

@@ -52,4 +52,7 @@ data class MessageStickerContent(
          * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
          */
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
-) : MessageImageInfoContent
+) : MessageImageInfoContent {
+        override val mimeType: String?
+                get() = encryptedFileInfo?.mimetype ?: info?.mimeType
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageVideoContent.kt

@@ -51,4 +51,7 @@ data class MessageVideoContent(
          * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
          */
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
-) : MessageWithAttachmentContent
+) : MessageWithAttachmentContent {
+        override val mimeType: String?
+                get() = encryptedFileInfo?.mimetype ?: videoInfo?.mimeType
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageWithAttachmentContent.kt

@@ -31,9 +31,13 @@ interface MessageWithAttachmentContent : MessageContent {
      * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
      */
     val encryptedFileInfo: EncryptedFileInfo?
+
+    val mimeType: String?
 }
 
 /**
  * Get the url of the encrypted file or of the file
  */
 fun MessageWithAttachmentContent.getFileUrl() = encryptedFileInfo?.url ?: url
+
+fun MessageWithAttachmentContent.getFileName() = (this as? MessageFileContent)?.getFileName() ?: body

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/powerlevels/PowerLevelsHelper.kt

@@ -17,6 +17,7 @@
 
 package im.vector.matrix.android.api.session.room.powerlevels
 
+import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.room.model.PowerLevelsContent
 
 /**
@@ -123,4 +124,59 @@ class PowerLevelsHelper(private val powerLevelsContent: PowerLevelsContent) {
             else      -> Role.Moderator.value
         }
     }
+
+    /**
+     * Check if user have the necessary power level to change room name
+     * @param userId the id of the user to check for.
+     * @return true if able to change room name
+     */
+    fun isUserAbleToChangeRoomName(userId: String): Boolean {
+        val powerLevel = getUserPowerLevelValue(userId)
+        val minPowerLevel = powerLevelsContent.events[EventType.STATE_ROOM_NAME] ?: powerLevelsContent.stateDefault
+        return powerLevel >= minPowerLevel
+    }
+
+    /**
+     * Check if user have the necessary power level to change room topic
+     * @param userId the id of the user to check for.
+     * @return true if able to change room topic
+     */
+    fun isUserAbleToChangeRoomTopic(userId: String): Boolean {
+        val powerLevel = getUserPowerLevelValue(userId)
+        val minPowerLevel = powerLevelsContent.events[EventType.STATE_ROOM_TOPIC] ?: powerLevelsContent.stateDefault
+        return powerLevel >= minPowerLevel
+    }
+
+    /**
+     * Check if user have the necessary power level to change room canonical alias
+     * @param userId the id of the user to check for.
+     * @return true if able to change room canonical alias
+     */
+    fun isUserAbleToChangeRoomCanonicalAlias(userId: String): Boolean {
+        val powerLevel = getUserPowerLevelValue(userId)
+        val minPowerLevel = powerLevelsContent.events[EventType.STATE_ROOM_CANONICAL_ALIAS] ?: powerLevelsContent.stateDefault
+        return powerLevel >= minPowerLevel
+    }
+
+    /**
+     * Check if user have the necessary power level to change room history readability
+     * @param userId the id of the user to check for.
+     * @return true if able to change room history readability
+     */
+    fun isUserAbleToChangeRoomHistoryReadability(userId: String): Boolean {
+        val powerLevel = getUserPowerLevelValue(userId)
+        val minPowerLevel = powerLevelsContent.events[EventType.STATE_ROOM_HISTORY_VISIBILITY] ?: powerLevelsContent.stateDefault
+        return powerLevel >= minPowerLevel
+    }
+
+    /**
+     * Check if user have the necessary power level to change room avatar
+     * @param userId the id of the user to check for.
+     * @return true if able to change room avatar
+     */
+    fun isUserAbleToChangeRoomAvatar(userId: String): Boolean {
+        val powerLevel = getUserPowerLevelValue(userId)
+        val minPowerLevel = powerLevelsContent.events[EventType.STATE_ROOM_AVATAR] ?: powerLevelsContent.stateDefault
+        return powerLevel >= minPowerLevel
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/state/StateService.kt

@@ -16,10 +16,12 @@
 
 package im.vector.matrix.android.api.session.room.state
 
+import android.net.Uri
 import androidx.lifecycle.LiveData
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.query.QueryStringValue
 import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.api.session.room.model.RoomHistoryVisibility
 import im.vector.matrix.android.api.util.Cancelable
 import im.vector.matrix.android.api.util.JsonDict
 import im.vector.matrix.android.api.util.Optional
@@ -31,6 +33,31 @@ interface StateService {
      */
     fun updateTopic(topic: String, callback: MatrixCallback<Unit>): Cancelable
 
+    /**
+     * Update the name of the room
+     */
+    fun updateName(name: String, callback: MatrixCallback<Unit>): Cancelable
+
+    /**
+     * Add new alias to the room.
+     */
+    fun addRoomAlias(roomAlias: String, callback: MatrixCallback<Unit>): Cancelable
+
+    /**
+     * Update the canonical alias of the room
+     */
+    fun updateCanonicalAlias(alias: String, callback: MatrixCallback<Unit>): Cancelable
+
+    /**
+     * Update the history readability of the room
+     */
+    fun updateHistoryReadability(readability: RoomHistoryVisibility, callback: MatrixCallback<Unit>): Cancelable
+
+    /**
+     * Update the avatar of the room
+     */
+    fun updateAvatar(avatarUri: Uri, fileName: String, callback: MatrixCallback<Unit>): Cancelable
+
     fun sendStateEvent(eventType: String, stateKey: String?, body: JsonDict, callback: MatrixCallback<Unit>): Cancelable
 
     fun getStateEvent(eventType: String, stateKey: QueryStringValue = QueryStringValue.NoCondition): Event?

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/typing/TypingUsersTracker.kt

@@ -16,7 +16,6 @@
 
 package im.vector.matrix.android.api.session.typing
 
-import androidx.lifecycle.LiveData
 import im.vector.matrix.android.api.session.room.sender.SenderInfo
 
 /**
@@ -29,9 +28,4 @@ interface TypingUsersTracker {
      * Returns the sender information of all currently typing users in a room, excluding yourself.
      */
     fun getTypingUsers(roomId: String): List<SenderInfo>
-
-    /**
-     * Returns a LiveData of the sender information of all currently typing users in a room, excluding yourself.
-     */
-    fun getTypingUsersLive(roomId: String): LiveData<List<SenderInfo>>
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/auth/DefaultAuthenticationService.kt

@@ -43,6 +43,8 @@ import im.vector.matrix.android.internal.auth.version.isSupportedBySdk
 import im.vector.matrix.android.internal.di.Unauthenticated
 import im.vector.matrix.android.internal.network.RetrofitFactory
 import im.vector.matrix.android.internal.network.executeRequest
+import im.vector.matrix.android.internal.network.httpclient.addSocketFactory
+import im.vector.matrix.android.internal.network.ssl.UnrecognizedCertificateException
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.configureWith
 import im.vector.matrix.android.internal.task.launchToCallback
@@ -121,7 +123,11 @@ internal class DefaultAuthenticationService @Inject constructor(
                         callback.onSuccess(it)
                     },
                     {
-                        callback.onFailure(it)
+                        if (it is UnrecognizedCertificateException) {
+                            callback.onFailure(Failure.UnrecognizedCertificateFailure(homeServerConnectionConfig.homeServerUri.toString(), it.fingerprint))
+                        } else {
+                            callback.onFailure(it)
+                        }
                     }
             )
         }
@@ -209,7 +215,7 @@ internal class DefaultAuthenticationService @Inject constructor(
 
         // Create a fake userId, for the getWellknown task
         val fakeUserId = "@alice:$domain"
-        val wellknownResult = getWellknownTask.execute(GetWellknownTask.Params(fakeUserId))
+        val wellknownResult = getWellknownTask.execute(GetWellknownTask.Params(fakeUserId, homeServerConnectionConfig))
 
         return when (wellknownResult) {
             is WellknownResult.Prompt -> {
@@ -248,7 +254,7 @@ internal class DefaultAuthenticationService @Inject constructor(
                 ?: let {
                     pendingSessionData?.homeServerConnectionConfig?.let {
                         DefaultRegistrationWizard(
-                                okHttpClient,
+                                buildClient(it),
                                 retrofitFactory,
                                 coroutineDispatchers,
                                 sessionCreator,
@@ -269,7 +275,7 @@ internal class DefaultAuthenticationService @Inject constructor(
                 ?: let {
                     pendingSessionData?.homeServerConnectionConfig?.let {
                         DefaultLoginWizard(
-                                okHttpClient,
+                                buildClient(it),
                                 retrofitFactory,
                                 coroutineDispatchers,
                                 sessionCreator,
@@ -321,9 +327,11 @@ internal class DefaultAuthenticationService @Inject constructor(
         }
     }
 
-    override fun getWellKnownData(matrixId: String, callback: MatrixCallback<WellknownResult>): Cancelable {
+    override fun getWellKnownData(matrixId: String,
+                                  homeServerConnectionConfig: HomeServerConnectionConfig?,
+                                  callback: MatrixCallback<WellknownResult>): Cancelable {
         return getWellknownTask
-                .configureWith(GetWellknownTask.Params(matrixId)) {
+                .configureWith(GetWellknownTask.Params(matrixId, homeServerConnectionConfig)) {
                     this.callback = callback
                 }
                 .executeBy(taskExecutor)
@@ -347,7 +355,14 @@ internal class DefaultAuthenticationService @Inject constructor(
     }
 
     private fun buildAuthAPI(homeServerConnectionConfig: HomeServerConnectionConfig): AuthAPI {
-        val retrofit = retrofitFactory.create(okHttpClient, homeServerConnectionConfig.homeServerUri.toString())
+        val retrofit = retrofitFactory.create(buildClient(homeServerConnectionConfig), homeServerConnectionConfig.homeServerUri.toString())
         return retrofit.create(AuthAPI::class.java)
     }
+
+    private fun buildClient(homeServerConnectionConfig: HomeServerConnectionConfig): OkHttpClient {
+        return okHttpClient.get()
+                .newBuilder()
+                .addSocketFactory(homeServerConnectionConfig)
+                .build()
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/auth/login/DefaultLoginWizard.kt

@@ -17,7 +17,6 @@
 package im.vector.matrix.android.internal.auth.login
 
 import android.util.Patterns
-import dagger.Lazy
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.api.auth.login.LoginWizard
@@ -44,7 +43,7 @@ import kotlinx.coroutines.withContext
 import okhttp3.OkHttpClient
 
 internal class DefaultLoginWizard(
-        okHttpClient: Lazy<OkHttpClient>,
+        okHttpClient: OkHttpClient,
         retrofitFactory: RetrofitFactory,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val sessionCreator: SessionCreator,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/auth/login/DirectLoginTask.kt

@@ -19,6 +19,7 @@ package im.vector.matrix.android.internal.auth.login
 import dagger.Lazy
 import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.api.auth.data.HomeServerConnectionConfig
+import im.vector.matrix.android.api.failure.Failure
 import im.vector.matrix.android.api.session.Session
 import im.vector.matrix.android.internal.auth.AuthAPI
 import im.vector.matrix.android.internal.auth.SessionCreator
@@ -26,6 +27,8 @@ import im.vector.matrix.android.internal.auth.data.PasswordLoginParams
 import im.vector.matrix.android.internal.di.Unauthenticated
 import im.vector.matrix.android.internal.network.RetrofitFactory
 import im.vector.matrix.android.internal.network.executeRequest
+import im.vector.matrix.android.internal.network.httpclient.addSocketFactory
+import im.vector.matrix.android.internal.network.ssl.UnrecognizedCertificateException
 import im.vector.matrix.android.internal.task.Task
 import okhttp3.OkHttpClient
 import javax.inject.Inject
@@ -47,15 +50,38 @@ internal class DefaultDirectLoginTask @Inject constructor(
 ) : DirectLoginTask {
 
     override suspend fun execute(params: DirectLoginTask.Params): Session {
-        val authAPI = retrofitFactory.create(okHttpClient, params.homeServerConnectionConfig.homeServerUri.toString())
+        val client = buildClient(params.homeServerConnectionConfig)
+        val homeServerUrl = params.homeServerConnectionConfig.homeServerUri.toString()
+
+        val authAPI = retrofitFactory.create(client, homeServerUrl)
                 .create(AuthAPI::class.java)
 
         val loginParams = PasswordLoginParams.userIdentifier(params.userId, params.password, params.deviceName)
 
-        val credentials = executeRequest<Credentials>(null) {
-            apiCall = authAPI.login(loginParams)
+        val credentials = try {
+            executeRequest<Credentials>(null) {
+                apiCall = authAPI.login(loginParams)
+            }
+        } catch (throwable: Throwable) {
+            when (throwable) {
+                is UnrecognizedCertificateException -> {
+                    throw Failure.UnrecognizedCertificateFailure(
+                            homeServerUrl,
+                            throwable.fingerprint
+                    )
+                }
+                else                                ->
+                    throw throwable
+            }
         }
 
         return sessionCreator.createSession(credentials, params.homeServerConnectionConfig)
     }
+
+    private fun buildClient(homeServerConnectionConfig: HomeServerConnectionConfig): OkHttpClient {
+        return okHttpClient.get()
+                .newBuilder()
+                .addSocketFactory(homeServerConnectionConfig)
+                .build()
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/auth/registration/DefaultRegistrationWizard.kt

@@ -16,7 +16,6 @@
 
 package im.vector.matrix.android.internal.auth.registration
 
-import dagger.Lazy
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.auth.data.LoginFlowTypes
 import im.vector.matrix.android.api.auth.registration.RegisterThreePid
@@ -41,7 +40,7 @@ import okhttp3.OkHttpClient
  * This class execute the registration request and is responsible to keep the session of interactive authentication
  */
 internal class DefaultRegistrationWizard(
-        private val okHttpClient: Lazy<OkHttpClient>,
+        private val okHttpClient: OkHttpClient,
         private val retrofitFactory: RetrofitFactory,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val sessionCreator: SessionCreator,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/DefaultCryptoService.kt

@@ -52,6 +52,7 @@ import im.vector.matrix.android.internal.crypto.actions.MegolmSessionDataImporte
 import im.vector.matrix.android.internal.crypto.actions.MessageEncrypter
 import im.vector.matrix.android.internal.crypto.actions.SetDeviceVerificationAction
 import im.vector.matrix.android.internal.crypto.algorithms.IMXEncrypting
+import im.vector.matrix.android.internal.crypto.algorithms.IMXWithHeldExtension
 import im.vector.matrix.android.internal.crypto.algorithms.megolm.MXMegolmEncryptionFactory
 import im.vector.matrix.android.internal.crypto.algorithms.olm.MXOlmEncryptionFactory
 import im.vector.matrix.android.internal.crypto.crosssigning.DefaultCrossSigningService
@@ -65,10 +66,10 @@ import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
 import im.vector.matrix.android.internal.crypto.model.event.EncryptedEventContent
 import im.vector.matrix.android.internal.crypto.model.event.OlmEventContent
 import im.vector.matrix.android.internal.crypto.model.event.RoomKeyContent
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
 import im.vector.matrix.android.internal.crypto.model.event.SecretSendEventContent
 import im.vector.matrix.android.internal.crypto.model.rest.DeviceInfo
 import im.vector.matrix.android.internal.crypto.model.rest.DevicesListResponse
-import im.vector.matrix.android.internal.crypto.model.rest.KeysUploadResponse
 import im.vector.matrix.android.internal.crypto.model.rest.RoomKeyRequestBody
 import im.vector.matrix.android.internal.crypto.model.toRest
 import im.vector.matrix.android.internal.crypto.repository.WarnOnUnknownDeviceRepository
@@ -96,6 +97,7 @@ import im.vector.matrix.android.internal.session.sync.model.SyncResponse
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.TaskThread
 import im.vector.matrix.android.internal.task.configureWith
+import im.vector.matrix.android.internal.task.launchToCallback
 import im.vector.matrix.android.internal.util.JsonCanonicalizer
 import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers
 import im.vector.matrix.android.internal.util.fetchCopied
@@ -338,11 +340,14 @@ internal class DefaultCryptoService @Inject constructor(
     }
 
     fun ensureDevice() {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        cryptoCoroutineScope.launchToCallback(coroutineDispatchers.crypto, NoOpMatrixCallback()) {
             // Open the store
             cryptoStore.open()
-            // TODO why do that everytime? we should mark that it was done
-            uploadDeviceKeys()
+            // this can throw if no network
+            tryThis {
+                uploadDeviceKeys()
+            }
+
             oneTimeKeysUploader.maybeUploadOneTimeKeys()
             // this can throw if no backup
             tryThis {
@@ -387,7 +392,7 @@ internal class DefaultCryptoService @Inject constructor(
 //            } else {
 
             // Why would we do that? it will be called at end of syn
-                incomingGossipingRequestManager.processReceivedGossipingRequests()
+            incomingGossipingRequestManager.processReceivedGossipingRequests()
 //            }
         }.fold(
                 {
@@ -807,6 +812,9 @@ internal class DefaultCryptoService @Inject constructor(
                     cryptoStore.saveGossipingEvent(event)
                     onSecretSendReceived(event)
                 }
+                EventType.ROOM_KEY_WITHHELD                      -> {
+                    onKeyWithHeldReceived(event)
+                }
                 else                                             -> {
                     // ignore
                 }
@@ -834,6 +842,20 @@ internal class DefaultCryptoService @Inject constructor(
         alg.onRoomKeyEvent(event, keysBackupService)
     }
 
+    private fun onKeyWithHeldReceived(event: Event) {
+        val withHeldContent = event.getClearContent().toModel<RoomKeyWithHeldContent>() ?: return Unit.also {
+            Timber.e("## CRYPTO | Malformed onKeyWithHeldReceived() : missing fields")
+        }
+        Timber.d("## CRYPTO | onKeyWithHeldReceived() received : content <$withHeldContent>")
+        val alg = roomDecryptorProvider.getOrCreateRoomDecryptor(withHeldContent.roomId, withHeldContent.algorithm)
+        if (alg is IMXWithHeldExtension) {
+            alg.onRoomKeyWithHeldEvent(withHeldContent)
+        } else {
+            Timber.e("## CRYPTO | onKeyWithHeldReceived() : Unable to handle WithHeldContent for ${withHeldContent.algorithm}")
+            return
+        }
+    }
+
     private fun onSecretSendReceived(event: Event) {
         Timber.i("## CRYPTO | GOSSIP onSecretSend() : onSecretSendReceived ${event.content?.get("sender_key")}")
         if (!event.isEncrypted()) {
@@ -869,7 +891,7 @@ internal class DefaultCryptoService @Inject constructor(
      */
     private fun handleSDKLevelGossip(secretName: String?, secretValue: String): Boolean {
         return when (secretName) {
-            MASTER_KEY_SSSS_NAME -> {
+            MASTER_KEY_SSSS_NAME       -> {
                 crossSigningService.onSecretMSKGossip(secretValue)
                 true
             }
@@ -961,7 +983,11 @@ internal class DefaultCryptoService @Inject constructor(
     /**
      * Upload my user's device keys.
      */
-    private suspend fun uploadDeviceKeys(): KeysUploadResponse {
+    private suspend fun uploadDeviceKeys() {
+        if (cryptoStore.getDeviceKeysUploaded()) {
+            Timber.d("Keys already uploaded, nothing to do")
+            return
+        }
         // Prepare the device keys data to send
         // Sign it
         val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, getMyDevice().signalableJSONDictionary())
@@ -972,7 +998,9 @@ internal class DefaultCryptoService @Inject constructor(
         )
 
         val uploadDeviceKeysParams = UploadKeysTask.Params(rest, null)
-        return uploadKeysTask.execute(uploadDeviceKeysParams)
+        uploadKeysTask.execute(uploadDeviceKeysParams)
+
+        cryptoStore.setDeviceKeysUploaded(true)
     }
 
     /**
@@ -1197,7 +1225,7 @@ internal class DefaultCryptoService @Inject constructor(
 //            }
             roomDecryptorProvider
                     .getOrCreateRoomDecryptor(event.roomId, wireContent.algorithm)
-                    ?.requestKeysForEvent(event) ?: run {
+                    ?.requestKeysForEvent(event, false) ?: run {
                 Timber.v("## CRYPTO | requestRoomKeyForEvent() : No room decryptor for roomId:${event.roomId} algorithm:${wireContent.algorithm}")
             }
         }
@@ -1311,6 +1339,13 @@ internal class DefaultCryptoService @Inject constructor(
         return cryptoStore.getGossipingEventsTrail()
     }
 
+    override fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int> {
+        return cryptoStore.getSharedWithInfo(roomId, sessionId)
+    }
+
+    override fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? {
+        return cryptoStore.getWithHeldMegolmSession(roomId, sessionId)
+    }
     /* ==========================================================================================
      * For test only
      * ========================================================================================== */

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/IMXDecrypting.kt

@@ -71,5 +71,5 @@ internal interface IMXDecrypting {
 
     fun shareSecretWithDevice(request: IncomingSecretShareRequest, secretValue : String) {}
 
-    fun requestKeysForEvent(event: Event)
+    fun requestKeysForEvent(event: Event, withHeld: Boolean)
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/IMXWithHeldExtension.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.algorithms
+
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
+
+internal interface IMXWithHeldExtension {
+    fun onRoomKeyWithHeldEvent(withHeldInfo: RoomKeyWithHeldContent)
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/megolm/MXMegolmDecryption.kt

@@ -30,10 +30,12 @@ import im.vector.matrix.android.internal.crypto.OutgoingGossipingRequestManager
 import im.vector.matrix.android.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
 import im.vector.matrix.android.internal.crypto.actions.MessageEncrypter
 import im.vector.matrix.android.internal.crypto.algorithms.IMXDecrypting
+import im.vector.matrix.android.internal.crypto.algorithms.IMXWithHeldExtension
 import im.vector.matrix.android.internal.crypto.keysbackup.DefaultKeysBackupService
 import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
 import im.vector.matrix.android.internal.crypto.model.event.EncryptedEventContent
 import im.vector.matrix.android.internal.crypto.model.event.RoomKeyContent
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
 import im.vector.matrix.android.internal.crypto.model.rest.ForwardedRoomKeyContent
 import im.vector.matrix.android.internal.crypto.model.rest.RoomKeyRequestBody
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
@@ -53,7 +55,7 @@ internal class MXMegolmDecryption(private val userId: String,
                                   private val sendToDeviceTask: SendToDeviceTask,
                                   private val coroutineDispatchers: MatrixCoroutineDispatchers,
                                   private val cryptoCoroutineScope: CoroutineScope
-) : IMXDecrypting {
+) : IMXDecrypting, IMXWithHeldExtension {
 
     var newSessionListener: NewSessionListener? = null
 
@@ -61,7 +63,7 @@ internal class MXMegolmDecryption(private val userId: String,
      * Events which we couldn't decrypt due to unknown sessions / indexes: map from
      * senderKey|sessionId to timelines to list of MatrixEvents.
      */
-    private var pendingEvents: MutableMap<String /* senderKey|sessionId */, MutableMap<String /* timelineId */, MutableList<Event>>> = HashMap()
+//    private var pendingEvents: MutableMap<String /* senderKey|sessionId */, MutableMap<String /* timelineId */, MutableList<Event>>> = HashMap()
 
     @Throws(MXCryptoError::class)
     override fun decryptEvent(event: Event, timeline: String): MXEventDecryptionResult {
@@ -113,9 +115,21 @@ internal class MXMegolmDecryption(private val userId: String,
                             if (throwable is MXCryptoError.OlmError) {
                                 // TODO Check the value of .message
                                 if (throwable.olmException.message == "UNKNOWN_MESSAGE_INDEX") {
-                                    addEventToPendingList(event, timeline)
+                                    // addEventToPendingList(event, timeline)
+                                    // The session might has been partially withheld (and only pass ratcheted)
+                                    val withHeldInfo = cryptoStore.getWithHeldMegolmSession(event.roomId, encryptedEventContent.sessionId)
+                                    if (withHeldInfo != null) {
+                                        if (requestKeysOnFail) {
+                                            requestKeysForEvent(event, true)
+                                        }
+                                        // Encapsulate as withHeld exception
+                                        throw MXCryptoError.Base(MXCryptoError.ErrorType.KEYS_WITHHELD,
+                                                withHeldInfo.code?.value ?: "",
+                                                withHeldInfo.reason)
+                                    }
+
                                     if (requestKeysOnFail) {
-                                        requestKeysForEvent(event)
+                                        requestKeysForEvent(event, false)
                                     }
                                 }
 
@@ -128,10 +142,25 @@ internal class MXMegolmDecryption(private val userId: String,
                                         detailedReason)
                             }
                             if (throwable is MXCryptoError.Base) {
-                                if (throwable.errorType == MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID) {
-                                    addEventToPendingList(event, timeline)
-                                    if (requestKeysOnFail) {
-                                        requestKeysForEvent(event)
+                                if (
+                                /** if the session is unknown*/
+                                        throwable.errorType == MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID
+                                ) {
+                                    val withHeldInfo = cryptoStore.getWithHeldMegolmSession(event.roomId, encryptedEventContent.sessionId)
+                                    if (withHeldInfo != null) {
+                                        if (requestKeysOnFail) {
+                                            requestKeysForEvent(event, true)
+                                        }
+                                        // Encapsulate as withHeld exception
+                                        throw MXCryptoError.Base(MXCryptoError.ErrorType.KEYS_WITHHELD,
+                                                withHeldInfo.code?.value ?: "",
+                                                withHeldInfo.reason)
+                                    } else {
+                                        // This is un-used in riotX SDK, not sure if needed
+                                        // addEventToPendingList(event, timeline)
+                                        if (requestKeysOnFail) {
+                                            requestKeysForEvent(event, false)
+                                        }
                                     }
                                 }
                             }
@@ -147,12 +176,12 @@ internal class MXMegolmDecryption(private val userId: String,
      *
      * @param event the event
      */
-    override fun requestKeysForEvent(event: Event) {
+    override fun requestKeysForEvent(event: Event, withHeld: Boolean) {
         val sender = event.senderId ?: return
         val encryptedEventContent = event.content.toModel<EncryptedEventContent>()
         val senderDevice = encryptedEventContent?.deviceId ?: return
 
-        val recipients = if (event.senderId == userId) {
+        val recipients = if (event.senderId == userId || withHeld) {
             mapOf(
                     userId to listOf("*")
             )
@@ -176,25 +205,25 @@ internal class MXMegolmDecryption(private val userId: String,
         outgoingGossipingRequestManager.sendRoomKeyRequest(requestBody, recipients)
     }
 
-    /**
-     * Add an event to the list of those we couldn't decrypt the first time we
-     * saw them.
-     *
-     * @param event      the event to try to decrypt later
-     * @param timelineId the timeline identifier
-     */
-    private fun addEventToPendingList(event: Event, timelineId: String) {
-        val encryptedEventContent = event.content.toModel<EncryptedEventContent>() ?: return
-        val pendingEventsKey = "${encryptedEventContent.senderKey}|${encryptedEventContent.sessionId}"
-
-        val timeline = pendingEvents.getOrPut(pendingEventsKey) { HashMap() }
-        val events = timeline.getOrPut(timelineId) { ArrayList() }
-
-        if (event !in events) {
-            Timber.v("## CRYPTO | addEventToPendingList() : add Event ${event.eventId} in room id ${event.roomId}")
-            events.add(event)
-        }
-    }
+//    /**
+//     * Add an event to the list of those we couldn't decrypt the first time we
+//     * saw them.
+//     *
+//     * @param event      the event to try to decrypt later
+//     * @param timelineId the timeline identifier
+//     */
+//    private fun addEventToPendingList(event: Event, timelineId: String) {
+//        val encryptedEventContent = event.content.toModel<EncryptedEventContent>() ?: return
+//        val pendingEventsKey = "${encryptedEventContent.senderKey}|${encryptedEventContent.sessionId}"
+//
+//        val timeline = pendingEvents.getOrPut(pendingEventsKey) { HashMap() }
+//        val events = timeline.getOrPut(timelineId) { ArrayList() }
+//
+//        if (event !in events) {
+//            Timber.v("## CRYPTO | addEventToPendingList() : add Event ${event.eventId} in room id ${event.roomId}")
+//            events.add(event)
+//        }
+//    }
 
     /**
      * Handle a key event.
@@ -349,4 +378,10 @@ internal class MXMegolmDecryption(private val userId: String,
                     }
         }
     }
+
+    override fun onRoomKeyWithHeldEvent(withHeldInfo: RoomKeyWithHeldContent) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            cryptoStore.addWithHeldMegolmSession(withHeldInfo)
+        }
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/megolm/MXMegolmEncryption.kt

@@ -18,6 +18,7 @@
 
 package im.vector.matrix.android.internal.crypto.algorithms.megolm
 
+import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.api.session.crypto.MXCryptoError
 import im.vector.matrix.android.api.session.events.model.Content
@@ -31,9 +32,14 @@ import im.vector.matrix.android.internal.crypto.algorithms.IMXEncrypting
 import im.vector.matrix.android.internal.crypto.keysbackup.DefaultKeysBackupService
 import im.vector.matrix.android.internal.crypto.model.CryptoDeviceInfo
 import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
+import im.vector.matrix.android.internal.crypto.model.event.WithHeldCode
+import im.vector.matrix.android.internal.crypto.model.forEach
 import im.vector.matrix.android.internal.crypto.repository.WarnOnUnknownDeviceRepository
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import im.vector.matrix.android.internal.crypto.tasks.SendToDeviceTask
+import im.vector.matrix.android.internal.task.TaskExecutor
+import im.vector.matrix.android.internal.task.configureWith
 import im.vector.matrix.android.internal.util.JsonCanonicalizer
 import im.vector.matrix.android.internal.util.convertToUTF8
 import timber.log.Timber
@@ -49,7 +55,8 @@ internal class MXMegolmEncryption(
         private val credentials: Credentials,
         private val sendToDeviceTask: SendToDeviceTask,
         private val messageEncrypter: MessageEncrypter,
-        private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository
+        private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository,
+        private val taskExecutor: TaskExecutor
 ) : IMXEncrypting {
 
     // OutboundSessionInfo. Null if we haven't yet started setting one up. Note
@@ -69,9 +76,26 @@ internal class MXMegolmEncryption(
         val ts = System.currentTimeMillis()
         Timber.v("## CRYPTO | encryptEventContent : getDevicesInRoom")
         val devices = getDevicesInRoom(userIds)
-        Timber.v("## CRYPTO | encryptEventContent ${System.currentTimeMillis() - ts}: getDevicesInRoom ${devices.map}")
-        val outboundSession = ensureOutboundSession(devices)
+        Timber.v("## CRYPTO | encryptEventContent ${System.currentTimeMillis() - ts}: getDevicesInRoom ${devices.allowedDevices.map}")
+        val outboundSession = ensureOutboundSession(devices.allowedDevices)
+
         return encryptContent(outboundSession, eventType, eventContent)
+                .also {
+                    notifyWithheldForSession(devices.withHeldDevices, outboundSession)
+                }
+    }
+
+    private fun notifyWithheldForSession(devices: MXUsersDevicesMap<WithHeldCode>, outboundSession: MXOutboundSessionInfo) {
+        mutableListOf<Pair<UserDevice, WithHeldCode>>().apply {
+            devices.forEach { userId, deviceId, withheldCode ->
+                this.add(UserDevice(userId, deviceId) to withheldCode)
+            }
+        }.groupBy(
+                { it.second },
+                { it.first }
+        ).forEach { (code, targets) ->
+            notifyKeyWithHeld(targets, outboundSession.sessionId, olmDevice.deviceCurve25519Key, code)
+        }
     }
 
     override fun discardSessionKey() {
@@ -95,7 +119,7 @@ internal class MXMegolmEncryption(
 
         defaultKeysBackupService.maybeBackupKeys()
 
-        return MXOutboundSessionInfo(sessionId)
+        return MXOutboundSessionInfo(sessionId, SharedWithHelper(roomId, sessionId, cryptoStore))
     }
 
     /**
@@ -121,7 +145,7 @@ internal class MXMegolmEncryption(
             val deviceIds = devicesInRoom.getUserDeviceIds(userId)
             for (deviceId in deviceIds!!) {
                 val deviceInfo = devicesInRoom.getObject(userId, deviceId)
-                if (deviceInfo != null && null == safeSession.sharedWithDevices.getObject(userId, deviceId)) {
+                if (deviceInfo != null && !cryptoStore.wasSessionSharedWithUser(roomId, safeSession.sessionId, userId, deviceId).found) {
                     val devices = shareMap.getOrPut(userId) { ArrayList() }
                     devices.add(deviceInfo)
                 }
@@ -198,15 +222,17 @@ internal class MXMegolmEncryption(
                 if (sessionResult?.sessionId == null) {
                     // no session with this device, probably because there
                     // were no one-time keys.
-                    //
-                    // we could send them a to_device message anyway, as a
-                    // signal that they have missed out on the key sharing
-                    // message because of the lack of keys, but there's not
-                    // much point in that really; it will mostly serve to clog
-                    // up to_device inboxes.
-                    //
-                    // ensureOlmSessionsForUsers has already done the logging,
-                    // so just skip it.
+
+                    // MSC 2399
+                    // send withheld m.no_olm: an olm session could not be established.
+                    // This may happen, for example, if the sender was unable to obtain a one-time key from the recipient.
+                    notifyKeyWithHeld(
+                            listOf(UserDevice(userId, deviceID)),
+                            session.sessionId,
+                            olmDevice.deviceCurve25519Key,
+                            WithHeldCode.NO_OLM
+                    )
+
                     continue
                 }
                 Timber.v("## CRYPTO | shareUserDevicesKey() : Sharing keys with device $userId:$deviceID")
@@ -214,29 +240,59 @@ internal class MXMegolmEncryption(
                 haveTargets = true
             }
         }
+
+        // Add the devices we have shared with to session.sharedWithDevices.
+        // we deliberately iterate over devicesByUser (ie, the devices we
+        // attempted to share with) rather than the contentMap (those we did
+        // share with), because we don't want to try to claim a one-time-key
+        // for dead devices on every message.
+        for ((userId, devicesToShareWith) in devicesByUser) {
+            for ((deviceId) in devicesToShareWith) {
+                session.sharedWithHelper.markedSessionAsShared(userId, deviceId, chainIndex)
+            }
+        }
+
         if (haveTargets) {
             t0 = System.currentTimeMillis()
             Timber.v("## CRYPTO | shareUserDevicesKey() : has target")
             val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, contentMap)
-            sendToDeviceTask.execute(sendToDeviceParams)
-            Timber.v("## CRYPTO | shareUserDevicesKey() : sendToDevice succeeds after "
-                    + (System.currentTimeMillis() - t0) + " ms")
-
-            // Add the devices we have shared with to session.sharedWithDevices.
-            // we deliberately iterate over devicesByUser (ie, the devices we
-            // attempted to share with) rather than the contentMap (those we did
-            // share with), because we don't want to try to claim a one-time-key
-            // for dead devices on every message.
-            for ((userId, devicesToShareWith) in devicesByUser) {
-                for ((deviceId) in devicesToShareWith) {
-                    session.sharedWithDevices.setObject(userId, deviceId, chainIndex)
-                }
+            try {
+                sendToDeviceTask.execute(sendToDeviceParams)
+                Timber.v("## CRYPTO | shareUserDevicesKey() : sendToDevice succeeds after ${System.currentTimeMillis() - t0} ms")
+            } catch (failure: Throwable) {
+                // What to do here...
+                Timber.e("## CRYPTO | shareUserDevicesKey() : Failed to share session <${session.sessionId}> with $devicesByUser ")
             }
         } else {
             Timber.v("## CRYPTO | shareUserDevicesKey() : no need to sharekey")
         }
     }
 
+    private fun notifyKeyWithHeld(targets: List<UserDevice>, sessionId: String, senderKey: String?, code: WithHeldCode) {
+        val withHeldContent = RoomKeyWithHeldContent(
+                roomId = roomId,
+                senderKey = senderKey,
+                algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
+                sessionId = sessionId,
+                codeString = code.value
+        )
+        val params = SendToDeviceTask.Params(
+                EventType.ROOM_KEY_WITHHELD,
+                MXUsersDevicesMap<Any>().apply {
+                    targets.forEach {
+                        setObject(it.userId, it.deviceId, withHeldContent)
+                    }
+                }
+        )
+        sendToDeviceTask.configureWith(params) {
+            callback = object : MatrixCallback<Unit> {
+                override fun onFailure(failure: Throwable) {
+                    Timber.e("## CRYPTO | notifyKeyWithHeld() : Failed to notify withheld key for $targets session: $sessionId ")
+                }
+            }
+        }.executeBy(taskExecutor)
+    }
+
     /**
      * process the pending encryptions
      */
@@ -271,7 +327,7 @@ internal class MXMegolmEncryption(
      *
      * @param userIds  the user ids whose devices must be checked.
      */
-    private suspend fun getDevicesInRoom(userIds: List<String>): MXUsersDevicesMap<CryptoDeviceInfo> {
+    private suspend fun getDevicesInRoom(userIds: List<String>): DeviceInRoomInfo {
         // We are happy to use a cached version here: we assume that if we already
         // have a list of the user's devices, then we already share an e2e room
         // with them, which means that they will have announced any new devices via
@@ -280,7 +336,7 @@ internal class MXMegolmEncryption(
         val encryptToVerifiedDevicesOnly = cryptoStore.getGlobalBlacklistUnverifiedDevices()
                 || cryptoStore.getRoomsListBlacklistUnverifiedDevices().contains(roomId)
 
-        val devicesInRoom = MXUsersDevicesMap<CryptoDeviceInfo>()
+        val devicesInRoom = DeviceInRoomInfo()
         val unknownDevices = MXUsersDevicesMap<CryptoDeviceInfo>()
 
         for (userId in keys.userIds) {
@@ -294,10 +350,12 @@ internal class MXMegolmEncryption(
                 }
                 if (deviceInfo.isBlocked) {
                     // Remove any blocked devices
+                    devicesInRoom.withHeldDevices.setObject(userId, deviceId, WithHeldCode.BLACKLISTED)
                     continue
                 }
 
                 if (!deviceInfo.isVerified && encryptToVerifiedDevicesOnly) {
+                    devicesInRoom.withHeldDevices.setObject(userId, deviceId, WithHeldCode.UNVERIFIED)
                     continue
                 }
 
@@ -305,7 +363,7 @@ internal class MXMegolmEncryption(
                     // Don't bother sending to ourself
                     continue
                 }
-                devicesInRoom.setObject(userId, deviceId, deviceInfo)
+                devicesInRoom.allowedDevices.setObject(userId, deviceId, deviceInfo)
             }
         }
         if (unknownDevices.isEmpty) {
@@ -324,8 +382,12 @@ internal class MXMegolmEncryption(
                 .also { Timber.w("Device not found") }
 
         // Get the chain index of the key we previously sent this device
-        val chainIndex = outboundSession?.sharedWithDevices?.getObject(userId, deviceId)?.toLong() ?: return false
-                .also { Timber.w("[MXMegolmEncryption] reshareKey : ERROR : Never share megolm with this device") }
+        val chainIndex = outboundSession?.sharedWithHelper?.wasSharedWith(userId, deviceId) ?: return false
+                .also {
+                    // Send a room key with held
+                    notifyKeyWithHeld(listOf(UserDevice(userId, deviceId)), sessionId, senderKey, WithHeldCode.UNAUTHORISED)
+                    Timber.w("[MXMegolmEncryption] reshareKey : ERROR : Never share megolm with this device")
+                }
 
         val devicesByUser = mapOf(userId to listOf(deviceInfo))
         val usersDeviceMap = ensureOlmSessionsForDevicesAction.handle(devicesByUser)
@@ -343,7 +405,7 @@ internal class MXMegolmEncryption(
                 .fold(
                         {
                             // TODO
-                            payloadJson["content"] = it.exportKeys(chainIndex) ?: ""
+                            payloadJson["content"] = it.exportKeys(chainIndex.toLong()) ?: ""
                         },
                         {
                             // TODO
@@ -354,9 +416,24 @@ internal class MXMegolmEncryption(
         val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
         val sendToDeviceMap = MXUsersDevicesMap<Any>()
         sendToDeviceMap.setObject(userId, deviceId, encodedPayload)
-        Timber.v("## CRYPTO | CRYPTO | shareKeysWithDevice() : sending to $userId:$deviceId")
+        Timber.v("## CRYPTO | CRYPTO | reshareKey() : sending to $userId:$deviceId")
         val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
-        sendToDeviceTask.execute(sendToDeviceParams)
-        return true
+        return try {
+            sendToDeviceTask.execute(sendToDeviceParams)
+            true
+        } catch (failure: Throwable) {
+            Timber.v("## CRYPTO | CRYPTO | reshareKey() : fail to send <$sessionId> to $userId:$deviceId")
+            false
+        }
     }
+
+    data class DeviceInRoomInfo(
+            val allowedDevices: MXUsersDevicesMap<CryptoDeviceInfo> = MXUsersDevicesMap(),
+            val withHeldDevices: MXUsersDevicesMap<WithHeldCode> = MXUsersDevicesMap()
+    )
+
+    data class UserDevice(
+            val userId: String,
+            val deviceId: String
+    )
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/megolm/MXMegolmEncryptionFactory.kt

@@ -25,6 +25,7 @@ import im.vector.matrix.android.internal.crypto.keysbackup.DefaultKeysBackupServ
 import im.vector.matrix.android.internal.crypto.repository.WarnOnUnknownDeviceRepository
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import im.vector.matrix.android.internal.crypto.tasks.SendToDeviceTask
+import im.vector.matrix.android.internal.task.TaskExecutor
 import javax.inject.Inject
 
 internal class MXMegolmEncryptionFactory @Inject constructor(
@@ -36,7 +37,8 @@ internal class MXMegolmEncryptionFactory @Inject constructor(
         private val credentials: Credentials,
         private val sendToDeviceTask: SendToDeviceTask,
         private val messageEncrypter: MessageEncrypter,
-        private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository) {
+        private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository,
+        private val taskExecutor: TaskExecutor) {
 
     fun create(roomId: String): MXMegolmEncryption {
         return MXMegolmEncryption(
@@ -49,6 +51,8 @@ internal class MXMegolmEncryptionFactory @Inject constructor(
                 credentials,
                 sendToDeviceTask,
                 messageEncrypter,
-                warnOnUnknownDevicesRepository)
+                warnOnUnknownDevicesRepository,
+                taskExecutor
+        )
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/megolm/MXOutboundSessionInfo.kt

@@ -23,17 +23,14 @@ import timber.log.Timber
 
 internal class MXOutboundSessionInfo(
         // The id of the session
-        val sessionId: String) {
+        val sessionId: String,
+        val sharedWithHelper: SharedWithHelper) {
     // When the session was created
     private val creationTime = System.currentTimeMillis()
 
     // Number of times this session has been used
     var useCount: Int = 0
 
-    // Devices with which we have shared the session key
-    // userId -> {deviceId -> msgindex}
-    val sharedWithDevices: MXUsersDevicesMap<Int> = MXUsersDevicesMap()
-
     fun needsRotation(rotationPeriodMsgs: Int, rotationPeriodMs: Int): Boolean {
         var needsRotation = false
         val sessionLifetime = System.currentTimeMillis() - creationTime
@@ -53,6 +50,7 @@ internal class MXOutboundSessionInfo(
      * @return true if we have shared the session with devices which aren't in devicesInRoom.
      */
     fun sharedWithTooManyDevices(devicesInRoom: MXUsersDevicesMap<CryptoDeviceInfo>): Boolean {
+        val sharedWithDevices = sharedWithHelper.sharedWithDevices()
         val userIds = sharedWithDevices.userIds
 
         for (userId in userIds) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/megolm/SharedWithHelper.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.algorithms.megolm
+
+import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
+import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
+
+internal class SharedWithHelper(
+        private val roomId: String,
+        private val sessionId: String,
+        private val cryptoStore: IMXCryptoStore) {
+
+    fun sharedWithDevices(): MXUsersDevicesMap<Int> {
+        return cryptoStore.getSharedWithInfo(roomId, sessionId)
+    }
+
+    fun wasSharedWith(userId: String, deviceId: String): Int? {
+        return cryptoStore.wasSessionSharedWithUser(roomId, sessionId, userId, deviceId).chainIndex
+    }
+
+    fun markedSessionAsShared(userId: String, deviceId: String, chainIndex: Int) {
+        cryptoStore.markedSessionAsShared(roomId, sessionId, userId, deviceId, chainIndex)
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/algorithms/olm/MXOlmDecryption.kt

@@ -212,7 +212,7 @@ internal class MXOlmDecryption(
         return res["payload"]
     }
 
-    override fun requestKeysForEvent(event: Event) {
+    override fun requestKeysForEvent(event: Event, withHeld: Boolean) {
         // nop
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/model/MXUsersDevicesMap.kt

@@ -119,3 +119,13 @@ class MXUsersDevicesMap<E> {
         return "MXUsersDevicesMap $map"
     }
 }
+
+inline fun <T> MXUsersDevicesMap<T>.forEach(action: (String, String, T) -> Unit) {
+    userIds.forEach { userId ->
+        getUserDeviceIds(userId)?.forEach { deviceId ->
+            getObject(userId, deviceId)?.let {
+                action(userId, deviceId, it)
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/model/event/RoomKeyWithHeldContent.kt

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package im.vector.matrix.android.internal.crypto.model.event
+
+import com.squareup.moshi.Json
+import com.squareup.moshi.JsonClass
+
+/**
+ * Class representing an sharekey content
+ */
+@JsonClass(generateAdapter = true)
+data class RoomKeyWithHeldContent(
+
+        /**
+         * Required if code is not m.no_olm. The ID of the room that the session belongs to.
+         */
+        @Json(name = "room_id") val roomId: String? = null,
+
+        /**
+         * Required. The encryption algorithm that the key is for.
+         */
+        @Json(name = "algorithm") val algorithm: String? = null,
+
+        /**
+         *  Required if code is not m.no_olm. The ID of the session.
+         */
+        @Json(name = "session_id") val sessionId: String? = null,
+
+        /**
+         * Required. The key of the session creator.
+         */
+        @Json(name = "sender_key") val senderKey: String? = null,
+
+        /**
+         *  Required. A machine-readable code for why the key was not sent
+         */
+        @Json(name = "code") val codeString: String? = null,
+
+        /**
+         *  A human-readable reason for why the key was not sent. The receiving client should only use this string if it does not understand the code.
+         */
+        @Json(name = "reason") val reason: String? = null
+
+) {
+    val code: WithHeldCode?
+        get() {
+            return WithHeldCode.fromCode(codeString)
+        }
+}
+
+enum class WithHeldCode(val value: String) {
+    /**
+     * the user/device was blacklisted
+     */
+    BLACKLISTED("m.blacklisted"),
+    /**
+     * the user/devices is unverified
+     */
+    UNVERIFIED("m.unverified"),
+    /**
+     * the user/device is not allowed have the key. For example, this would usually be sent in response
+     * to a key request if the user was not in the room when the message was sent
+     */
+    UNAUTHORISED("m.unauthorised"),
+    /**
+     * Sent in reply to a key request if the device that the key is requested from does not have the requested key
+     */
+    UNAVAILABLE("m.unavailable"),
+    /**
+     * An olm session could not be established.
+     * This may happen, for example, if the sender was unable to obtain a one-time key from the recipient.
+     */
+    NO_OLM("m.no_olm");
+
+    companion object {
+        fun fromCode(code: String?): WithHeldCode? {
+            return when (code) {
+                BLACKLISTED.value  -> BLACKLISTED
+                UNVERIFIED.value   -> UNVERIFIED
+                UNAUTHORISED.value -> UNAUTHORISED
+                UNAVAILABLE.value  -> UNAVAILABLE
+                NO_OLM.value       -> NO_OLM
+                else               -> null
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/IMXCryptoStore.kt

@@ -1,3 +1,4 @@
+
 /*
  * Copyright 2016 OpenMarket Ltd
  * Copyright 2018 New Vector Ltd
@@ -30,8 +31,10 @@ import im.vector.matrix.android.internal.crypto.OutgoingRoomKeyRequest
 import im.vector.matrix.android.internal.crypto.OutgoingSecretRequest
 import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
 import im.vector.matrix.android.internal.crypto.model.CryptoDeviceInfo
+import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
 import im.vector.matrix.android.internal.crypto.model.OlmInboundGroupSessionWrapper2
 import im.vector.matrix.android.internal.crypto.model.OlmSessionWrapper
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
 import im.vector.matrix.android.internal.crypto.model.rest.DeviceInfo
 import im.vector.matrix.android.internal.crypto.model.rest.RoomKeyRequestBody
 import im.vector.matrix.android.internal.crypto.store.db.model.KeysBackupDataEntity
@@ -416,6 +419,13 @@ internal interface IMXCryptoStore {
 
     fun updateUsersTrust(check: (String) -> Boolean)
 
+    fun addWithHeldMegolmSession(withHeldContent: RoomKeyWithHeldContent)
+    fun getWithHeldMegolmSession(roomId: String, sessionId: String) : RoomKeyWithHeldContent?
+
+    fun markedSessionAsShared(roomId: String?, sessionId: String, userId: String, deviceId: String, chainIndex: Int)
+    fun wasSessionSharedWithUser(roomId: String?, sessionId: String, userId: String, deviceId: String) : SharedSessionResult
+    data class SharedSessionResult(val found: Boolean, val chainIndex: Int?)
+    fun getSharedWithInfo(roomId: String?, sessionId: String) : MXUsersDevicesMap<Int>
     // Dev tools
 
     fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest>
@@ -423,4 +433,7 @@ internal interface IMXCryptoStore {
     fun getOutgoingSecretRequest(secretName: String): OutgoingSecretRequest?
     fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
     fun getGossipingEventsTrail(): List<Event>
+
+    fun setDeviceKeysUploaded(uploaded: Boolean)
+    fun getDeviceKeysUploaded(): Boolean
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStore.kt

@@ -31,6 +31,7 @@ import im.vector.matrix.android.internal.crypto.GossipingRequestState
 import im.vector.matrix.android.internal.crypto.IncomingRoomKeyRequest
 import im.vector.matrix.android.internal.crypto.IncomingSecretShareRequest
 import im.vector.matrix.android.internal.crypto.IncomingShareRequestCommon
+import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import im.vector.matrix.android.internal.crypto.NewSessionListener
 import im.vector.matrix.android.internal.crypto.OutgoingGossipingRequestState
 import im.vector.matrix.android.internal.crypto.OutgoingRoomKeyRequest
@@ -38,8 +39,10 @@ import im.vector.matrix.android.internal.crypto.OutgoingSecretRequest
 import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
 import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
 import im.vector.matrix.android.internal.crypto.model.CryptoDeviceInfo
+import im.vector.matrix.android.internal.crypto.model.MXUsersDevicesMap
 import im.vector.matrix.android.internal.crypto.model.OlmInboundGroupSessionWrapper2
 import im.vector.matrix.android.internal.crypto.model.OlmSessionWrapper
+import im.vector.matrix.android.internal.crypto.model.event.RoomKeyWithHeldContent
 import im.vector.matrix.android.internal.crypto.model.rest.DeviceInfo
 import im.vector.matrix.android.internal.crypto.model.rest.RoomKeyRequestBody
 import im.vector.matrix.android.internal.crypto.model.toEntity
@@ -66,10 +69,13 @@ import im.vector.matrix.android.internal.crypto.store.db.model.OlmSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.OlmSessionEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.OutgoingGossipingRequestEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.OutgoingGossipingRequestEntityFields
+import im.vector.matrix.android.internal.crypto.store.db.model.SharedSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.TrustLevelEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.UserEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.UserEntityFields
+import im.vector.matrix.android.internal.crypto.store.db.model.WithHeldSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.createPrimaryKey
+import im.vector.matrix.android.internal.crypto.store.db.query.create
 import im.vector.matrix.android.internal.crypto.store.db.query.delete
 import im.vector.matrix.android.internal.crypto.store.db.query.get
 import im.vector.matrix.android.internal.crypto.store.db.query.getById
@@ -836,6 +842,18 @@ internal class RealmCryptoStore @Inject constructor(
         } ?: false
     }
 
+    override fun setDeviceKeysUploaded(uploaded: Boolean) {
+        doRealmTransaction(realmConfiguration) {
+            it.where<CryptoMetadataEntity>().findFirst()?.deviceKeysSentToServer = uploaded
+        }
+    }
+
+    override fun getDeviceKeysUploaded(): Boolean {
+        return doWithRealm(realmConfiguration) {
+            it.where<CryptoMetadataEntity>().findFirst()?.deviceKeysSentToServer
+        } ?: false
+    }
+
     override fun setRoomsListBlacklistUnverifiedDevices(roomIds: List<String>) {
         doRealmTransaction(realmConfiguration) {
             // Reset all
@@ -1427,4 +1445,68 @@ internal class RealmCryptoStore @Inject constructor(
             return existing
         }
     }
+
+    override fun addWithHeldMegolmSession(withHeldContent: RoomKeyWithHeldContent) {
+        val roomId = withHeldContent.roomId ?: return
+        val sessionId = withHeldContent.sessionId ?: return
+        if (withHeldContent.algorithm != MXCRYPTO_ALGORITHM_MEGOLM) return
+        doRealmTransaction(realmConfiguration) { realm ->
+            WithHeldSessionEntity.getOrCreate(realm, roomId, sessionId)?.let {
+                it.code = withHeldContent.code
+                it.senderKey = withHeldContent.senderKey
+                it.reason = withHeldContent.reason
+            }
+        }
+    }
+
+    override fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? {
+        return doWithRealm(realmConfiguration) { realm ->
+            WithHeldSessionEntity.get(realm, roomId, sessionId)?.let {
+                RoomKeyWithHeldContent(
+                        roomId = roomId,
+                        sessionId = sessionId,
+                        algorithm = it.algorithm,
+                        codeString = it.codeString,
+                        reason = it.reason,
+                        senderKey = it.senderKey
+                )
+            }
+        }
+    }
+
+    override fun markedSessionAsShared(roomId: String?, sessionId: String, userId: String, deviceId: String, chainIndex: Int) {
+        doRealmTransaction(realmConfiguration) { realm ->
+            SharedSessionEntity.create(
+                    realm = realm,
+                    roomId = roomId,
+                    sessionId = sessionId,
+                    userId = userId,
+                    deviceId = deviceId,
+                    chainIndex = chainIndex
+            )
+        }
+    }
+
+    override fun wasSessionSharedWithUser(roomId: String?, sessionId: String, userId: String, deviceId: String): IMXCryptoStore.SharedSessionResult {
+        return doWithRealm(realmConfiguration) { realm ->
+            SharedSessionEntity.get(realm, roomId, sessionId, userId, deviceId)?.let {
+                IMXCryptoStore.SharedSessionResult(true, it.chainIndex)
+            } ?: IMXCryptoStore.SharedSessionResult(false, null)
+        }
+    }
+
+    override fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int> {
+        return doWithRealm(realmConfiguration) { realm ->
+            val result = MXUsersDevicesMap<Int>()
+            SharedSessionEntity.get(realm, roomId, sessionId)
+                    .groupBy { it.userId }
+                    .forEach { (userId, shared) ->
+                        shared.forEach {
+                            result.setObject(userId, it.deviceId, it.chainIndex)
+                        }
+                    }
+
+            result
+        }
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -36,8 +36,10 @@ import im.vector.matrix.android.internal.crypto.store.db.model.MyDeviceLastSeenI
 import im.vector.matrix.android.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.OlmSessionEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.OutgoingGossipingRequestEntityFields
+import im.vector.matrix.android.internal.crypto.store.db.model.SharedSessionEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.TrustLevelEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.UserEntityFields
+import im.vector.matrix.android.internal.crypto.store.db.model.WithHeldSessionEntityFields
 import im.vector.matrix.android.internal.di.SerializeNulls
 import io.realm.DynamicRealm
 import io.realm.RealmMigration
@@ -52,7 +54,7 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
         // 0, 1, 2: legacy Riot-Android
         // 3: migrate to RiotX schema
         // 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6)
-        const val CRYPTO_STORE_SCHEMA_VERSION = 9L
+        const val CRYPTO_STORE_SCHEMA_VERSION = 11L
     }
 
     override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
@@ -67,6 +69,8 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
         if (oldVersion <= 6) migrateTo7(realm)
         if (oldVersion <= 7) migrateTo8(realm)
         if (oldVersion <= 8) migrateTo9(realm)
+        if (oldVersion <= 9) migrateTo10(realm)
+        if (oldVersion <= 10) migrateTo11(realm)
     }
 
     private fun migrateTo1Legacy(realm: DynamicRealm) {
@@ -173,13 +177,14 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                     }
                 }
 
-        // Convert MXOlmInboundGroupSession2 to OlmInboundGroupSessionWrapper2
+        // Convert MXOlmInboundGroupSession2 to OlmInboundGroupSessionWrapper
         realm.schema.get("OlmInboundGroupSessionEntity")
                 ?.transform { obj ->
                     try {
                         val oldSerializedData = obj.getString("olmInboundGroupSessionData")
                         deserializeFromRealm<MXOlmInboundGroupSession2>(oldSerializedData)?.let { mxOlmInboundGroupSession2 ->
-                            val newOlmInboundGroupSessionWrapper2 = OlmInboundGroupSessionWrapper2()
+                            val sessionKey = mxOlmInboundGroupSession2.mSession.sessionIdentifier()
+                            val newOlmInboundGroupSessionWrapper = OlmInboundGroupSessionWrapper(sessionKey, false)
                                     .apply {
                                         olmInboundGroupSession = mxOlmInboundGroupSession2.mSession
                                         roomId = mxOlmInboundGroupSession2.mRoomId
@@ -188,7 +193,7 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                                         forwardingCurve25519KeyChain = mxOlmInboundGroupSession2.mForwardingCurve25519KeyChain
                                     }
 
-                            obj.setString("olmInboundGroupSessionData", serializeForRealm(newOlmInboundGroupSessionWrapper2))
+                            obj.setString("olmInboundGroupSessionData", serializeForRealm(newOlmInboundGroupSessionWrapper))
                         }
                     } catch (e: Exception) {
                         Timber.e(e, "Error")
@@ -416,4 +421,37 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
             }
         }
     }
+
+    // Version 10L added WithHeld Keys Info (MSC2399)
+    private fun migrateTo10(realm: DynamicRealm) {
+        Timber.d("Step 9 -> 10")
+        realm.schema.create("WithHeldSessionEntity")
+                .addField(WithHeldSessionEntityFields.ROOM_ID, String::class.java)
+                .addField(WithHeldSessionEntityFields.ALGORITHM, String::class.java)
+                .addField(WithHeldSessionEntityFields.SESSION_ID, String::class.java)
+                .addIndex(WithHeldSessionEntityFields.SESSION_ID)
+                .addField(WithHeldSessionEntityFields.SENDER_KEY, String::class.java)
+                .addIndex(WithHeldSessionEntityFields.SENDER_KEY)
+                .addField(WithHeldSessionEntityFields.CODE_STRING, String::class.java)
+                .addField(WithHeldSessionEntityFields.REASON, String::class.java)
+
+        realm.schema.create("SharedSessionEntity")
+                .addField(SharedSessionEntityFields.ROOM_ID, String::class.java)
+                .addField(SharedSessionEntityFields.ALGORITHM, String::class.java)
+                .addField(SharedSessionEntityFields.SESSION_ID, String::class.java)
+                .addIndex(SharedSessionEntityFields.SESSION_ID)
+                .addField(SharedSessionEntityFields.USER_ID, String::class.java)
+                .addIndex(SharedSessionEntityFields.USER_ID)
+                .addField(SharedSessionEntityFields.DEVICE_ID, String::class.java)
+                .addIndex(SharedSessionEntityFields.DEVICE_ID)
+                .addField(SharedSessionEntityFields.CHAIN_INDEX, Long::class.java)
+                .setNullable(SharedSessionEntityFields.CHAIN_INDEX, true)
+    }
+
+    // Version 11L added deviceKeysSentToServer boolean to CryptoMetadataEntity
+    private fun migrateTo11(realm: DynamicRealm) {
+        Timber.d("Step 10 -> 11")
+        realm.schema.get("CryptoMetadataEntity")
+                ?.addField(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER, Boolean::class.java)
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStoreModule.kt

@@ -28,8 +28,10 @@ import im.vector.matrix.android.internal.crypto.store.db.model.MyDeviceLastSeenI
 import im.vector.matrix.android.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.OlmSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.OutgoingGossipingRequestEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.SharedSessionEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.TrustLevelEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.UserEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.WithHeldSessionEntity
 import io.realm.annotations.RealmModule
 
 /**
@@ -50,6 +52,8 @@ import io.realm.annotations.RealmModule
             GossipingEventEntity::class,
             IncomingGossipingRequestEntity::class,
             OutgoingGossipingRequestEntity::class,
-            MyDeviceLastSeenInfoEntity::class
+            MyDeviceLastSeenInfoEntity::class,
+            WithHeldSessionEntity::class,
+            SharedSessionEntity::class
         ])
 internal class RealmCryptoStoreModule

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/model/CryptoMetadataEntity.kt

@@ -36,6 +36,9 @@ internal open class CryptoMetadataEntity(
         // The keys backup version currently used. Null means no backup.
         var backupVersion: String? = null,
 
+        // The device keys has been sent to the homeserver
+        var deviceKeysSentToServer: Boolean = false,
+
         var xSignMasterPrivateKey: String? = null,
         var xSignUserPrivateKey: String? = null,
         var xSignSelfSignedPrivateKey: String? = null,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/model/SharedSessionEntity.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.model
+
+import io.realm.RealmObject
+import io.realm.annotations.Index
+
+/**
+ * Keep a record of to whom (user/device) a given session should have been shared.
+ * It will be used to reply to keyshare requests from other users, in order to see if
+ * this session was originaly shared with a given user
+ */
+internal open class SharedSessionEntity(
+        var roomId: String? = null,
+        var algorithm: String? = null,
+        @Index var sessionId: String? = null,
+        @Index var userId: String? = null,
+        @Index var deviceId: String? = null,
+        var chainIndex: Int? = null
+) : RealmObject() {
+
+    companion object
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/model/WithHeldSessionEntity.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.model
+
+import im.vector.matrix.android.internal.crypto.model.event.WithHeldCode
+import io.realm.RealmObject
+import io.realm.annotations.Index
+
+/**
+ * When an encrypted message is sent in a room, the megolm key might not be sent to all devices present in the room.
+ * Sometimes this may be inadvertent (for example, if the sending device is not aware of some devices that have joined),
+ * but some times, this may be purposeful.
+ * For example, the sender may have blacklisted certain devices or users,
+ * or may be choosing to not send the megolm key to devices that they have not verified yet.
+ */
+internal open class WithHeldSessionEntity(
+        var roomId: String? = null,
+        var algorithm: String? = null,
+        @Index var sessionId: String? = null,
+        @Index var senderKey: String? = null,
+        var codeString: String? = null,
+        var reason: String? = null
+) : RealmObject() {
+
+    var code: WithHeldCode?
+        get() {
+            return WithHeldCode.fromCode(codeString)
+        }
+        set(code) {
+            codeString = code?.value
+        }
+
+    companion object
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/query/SharedSessionQueries.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.query
+
+import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import im.vector.matrix.android.internal.crypto.store.db.model.SharedSessionEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.SharedSessionEntityFields
+import io.realm.Realm
+import io.realm.RealmResults
+import io.realm.kotlin.createObject
+import io.realm.kotlin.where
+
+internal fun SharedSessionEntity.Companion.get(realm: Realm, roomId: String?, sessionId: String, userId: String, deviceId: String)
+        : SharedSessionEntity? {
+    return realm.where<SharedSessionEntity>()
+            .equalTo(SharedSessionEntityFields.ROOM_ID, roomId)
+            .equalTo(SharedSessionEntityFields.SESSION_ID, sessionId)
+            .equalTo(SharedSessionEntityFields.ALGORITHM, MXCRYPTO_ALGORITHM_MEGOLM)
+            .equalTo(SharedSessionEntityFields.USER_ID, userId)
+            .equalTo(SharedSessionEntityFields.DEVICE_ID, deviceId)
+            .findFirst()
+}
+
+internal fun SharedSessionEntity.Companion.get(realm: Realm, roomId: String?, sessionId: String)
+        : RealmResults<SharedSessionEntity> {
+    return realm.where<SharedSessionEntity>()
+            .equalTo(SharedSessionEntityFields.ROOM_ID, roomId)
+            .equalTo(SharedSessionEntityFields.SESSION_ID, sessionId)
+            .equalTo(SharedSessionEntityFields.ALGORITHM, MXCRYPTO_ALGORITHM_MEGOLM)
+            .findAll()
+}
+
+internal fun SharedSessionEntity.Companion.create(realm: Realm, roomId: String?, sessionId: String, userId: String, deviceId: String, chainIndex: Int)
+        : SharedSessionEntity {
+    return realm.createObject<SharedSessionEntity>().apply {
+        this.roomId = roomId
+        this.algorithm = MXCRYPTO_ALGORITHM_MEGOLM
+        this.sessionId = sessionId
+        this.userId = userId
+        this.deviceId = deviceId
+        this.chainIndex = chainIndex
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/query/WithHeldSessionQueries.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.query
+
+import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import im.vector.matrix.android.internal.crypto.store.db.model.WithHeldSessionEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.WithHeldSessionEntityFields
+import io.realm.Realm
+import io.realm.kotlin.createObject
+import io.realm.kotlin.where
+
+internal fun WithHeldSessionEntity.Companion.get(realm: Realm, roomId: String, sessionId: String): WithHeldSessionEntity? {
+    return realm.where<WithHeldSessionEntity>()
+            .equalTo(WithHeldSessionEntityFields.ROOM_ID, roomId)
+            .equalTo(WithHeldSessionEntityFields.SESSION_ID, sessionId)
+            .equalTo(WithHeldSessionEntityFields.ALGORITHM, MXCRYPTO_ALGORITHM_MEGOLM)
+            .findFirst()
+}
+
+internal fun WithHeldSessionEntity.Companion.getOrCreate(realm: Realm, roomId: String, sessionId: String): WithHeldSessionEntity? {
+    return get(realm, roomId, sessionId)
+            ?: realm.createObject<WithHeldSessionEntity>().apply {
+                this.roomId = roomId
+                this.algorithm = MXCRYPTO_ALGORITHM_MEGOLM
+                this.sessionId = sessionId
+            }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/RoomVerificationUpdateTask.kt

@@ -1,161 +0,0 @@
-/*
- * Copyright 2019 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.crypto.tasks
-
-import im.vector.matrix.android.api.session.crypto.CryptoService
-import im.vector.matrix.android.api.session.crypto.MXCryptoError
-import im.vector.matrix.android.api.session.crypto.verification.VerificationService
-import im.vector.matrix.android.api.session.events.model.Event
-import im.vector.matrix.android.api.session.events.model.EventType
-import im.vector.matrix.android.api.session.events.model.toModel
-import im.vector.matrix.android.api.session.room.model.message.MessageContent
-import im.vector.matrix.android.api.session.room.model.message.MessageRelationContent
-import im.vector.matrix.android.api.session.room.model.message.MessageType
-import im.vector.matrix.android.api.session.room.model.message.MessageVerificationReadyContent
-import im.vector.matrix.android.api.session.room.model.message.MessageVerificationRequestContent
-import im.vector.matrix.android.api.session.room.model.message.MessageVerificationStartContent
-import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
-import im.vector.matrix.android.internal.crypto.verification.DefaultVerificationService
-import im.vector.matrix.android.internal.di.DeviceId
-import im.vector.matrix.android.internal.di.UserId
-import im.vector.matrix.android.internal.task.Task
-import timber.log.Timber
-import java.util.ArrayList
-import javax.inject.Inject
-
-internal interface RoomVerificationUpdateTask : Task<RoomVerificationUpdateTask.Params, Unit> {
-    data class Params(
-            val events: List<Event>,
-            val verificationService: DefaultVerificationService,
-            val cryptoService: CryptoService
-    )
-}
-
-internal class DefaultRoomVerificationUpdateTask @Inject constructor(
-        @UserId private val userId: String,
-        @DeviceId private val deviceId: String?,
-        private val cryptoService: CryptoService) : RoomVerificationUpdateTask {
-
-    companion object {
-        // XXX what about multi-account?
-        private val transactionsHandledByOtherDevice = ArrayList<String>()
-    }
-
-    override suspend fun execute(params: RoomVerificationUpdateTask.Params) {
-        // TODO ignore initial sync or back pagination?
-
-        params.events.forEach { event ->
-            Timber.v("## SAS Verification live observer: received msgId: ${event.eventId} msgtype: ${event.type} from ${event.senderId}")
-
-            // If the request is in the future by more than 5 minutes or more than 10 minutes in the past,
-            // the message should be ignored by the receiver.
-
-            if (!VerificationService.isValidRequest(event.ageLocalTs
-                            ?: event.originServerTs)) return@forEach Unit.also {
-                Timber.d("## SAS Verification live observer: msgId: ${event.eventId} is outdated")
-            }
-
-            // decrypt if needed?
-            if (event.isEncrypted() && event.mxDecryptionResult == null) {
-                // TODO use a global event decryptor? attache to session and that listen to new sessionId?
-                // for now decrypt sync
-                try {
-                    val result = cryptoService.decryptEvent(event, "")
-                    event.mxDecryptionResult = OlmDecryptionResult(
-                            payload = result.clearEvent,
-                            senderKey = result.senderCurve25519Key,
-                            keysClaimed = result.claimedEd25519Key?.let { mapOf("ed25519" to it) },
-                            forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
-                    )
-                } catch (e: MXCryptoError) {
-                    Timber.e("## SAS Failed to decrypt event: ${event.eventId}")
-                    params.verificationService.onPotentiallyInterestingEventRoomFailToDecrypt(event)
-                }
-            }
-            Timber.v("## SAS Verification live observer: received msgId: ${event.eventId} type: ${event.getClearType()}")
-
-            // Relates to is not encrypted
-            val relatesToEventId = event.content.toModel<MessageRelationContent>()?.relatesTo?.eventId
-
-            if (event.senderId == userId) {
-                // If it's send from me, we need to keep track of Requests or Start
-                // done from another device of mine
-
-                if (EventType.MESSAGE == event.getClearType()) {
-                    val msgType = event.getClearContent().toModel<MessageContent>()?.msgType
-                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == msgType) {
-                        event.getClearContent().toModel<MessageVerificationRequestContent>()?.let {
-                            if (it.fromDevice != deviceId) {
-                                // The verification is requested from another device
-                                Timber.v("## SAS Verification live observer: Transaction requested from other device  tid:${event.eventId} ")
-                                event.eventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
-                            }
-                        }
-                    }
-                } else if (EventType.KEY_VERIFICATION_START == event.getClearType()) {
-                    event.getClearContent().toModel<MessageVerificationStartContent>()?.let {
-                        if (it.fromDevice != deviceId) {
-                            // The verification is started from another device
-                            Timber.v("## SAS Verification live observer: Transaction started by other device  tid:$relatesToEventId ")
-                            relatesToEventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
-                            params.verificationService.onRoomRequestHandledByOtherDevice(event)
-                        }
-                    }
-                } else if (EventType.KEY_VERIFICATION_READY == event.getClearType()) {
-                    event.getClearContent().toModel<MessageVerificationReadyContent>()?.let {
-                        if (it.fromDevice != deviceId) {
-                            // The verification is started from another device
-                            Timber.v("## SAS Verification live observer: Transaction started by other device  tid:$relatesToEventId ")
-                            relatesToEventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
-                            params.verificationService.onRoomRequestHandledByOtherDevice(event)
-                        }
-                    }
-                } else if (EventType.KEY_VERIFICATION_CANCEL == event.getClearType() || EventType.KEY_VERIFICATION_DONE == event.getClearType()) {
-                    relatesToEventId?.let {
-                        transactionsHandledByOtherDevice.remove(it)
-                        params.verificationService.onRoomRequestHandledByOtherDevice(event)
-                    }
-                }
-
-                Timber.v("## SAS Verification ignoring message sent by me: ${event.eventId} type: ${event.getClearType()}")
-                return@forEach
-            }
-
-            if (relatesToEventId != null && transactionsHandledByOtherDevice.contains(relatesToEventId)) {
-                // Ignore this event, it is directed to another of my devices
-                Timber.v("## SAS Verification live observer: Ignore Transaction handled by other device  tid:$relatesToEventId ")
-                return@forEach
-            }
-            when (event.getClearType()) {
-                EventType.KEY_VERIFICATION_START,
-                EventType.KEY_VERIFICATION_ACCEPT,
-                EventType.KEY_VERIFICATION_KEY,
-                EventType.KEY_VERIFICATION_MAC,
-                EventType.KEY_VERIFICATION_CANCEL,
-                EventType.KEY_VERIFICATION_READY,
-                EventType.KEY_VERIFICATION_DONE -> {
-                    params.verificationService.onRoomEvent(event)
-                }
-                EventType.MESSAGE -> {
-                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.msgType) {
-                        params.verificationService.onRoomRequestReceived(event)
-                    }
-                }
-            }
-        }
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/verification/VerificationMessageLiveObserver.kt

@@ -1,73 +0,0 @@
-/*
- * Copyright 2019 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package im.vector.matrix.android.internal.crypto.verification
-
-import com.zhuinden.monarchy.Monarchy
-import im.vector.matrix.android.api.session.crypto.CryptoService
-import im.vector.matrix.android.api.session.events.model.EventType
-import im.vector.matrix.android.api.session.events.model.LocalEcho
-import im.vector.matrix.android.internal.crypto.tasks.DefaultRoomVerificationUpdateTask
-import im.vector.matrix.android.internal.crypto.tasks.RoomVerificationUpdateTask
-import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
-import im.vector.matrix.android.internal.database.mapper.asDomain
-import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.whereTypes
-import im.vector.matrix.android.internal.di.SessionDatabase
-import im.vector.matrix.android.internal.task.TaskExecutor
-import im.vector.matrix.android.internal.task.configureWith
-import io.realm.OrderedCollectionChangeSet
-import io.realm.RealmConfiguration
-import io.realm.RealmResults
-import javax.inject.Inject
-
-internal class VerificationMessageLiveObserver @Inject constructor(
-        @SessionDatabase realmConfiguration: RealmConfiguration,
-        private val roomVerificationUpdateTask: DefaultRoomVerificationUpdateTask,
-        private val cryptoService: CryptoService,
-        private val verificationService: DefaultVerificationService,
-        private val taskExecutor: TaskExecutor
-) : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
-
-    override val query = Monarchy.Query {
-        EventEntity.whereTypes(it, listOf(
-                EventType.KEY_VERIFICATION_START,
-                EventType.KEY_VERIFICATION_ACCEPT,
-                EventType.KEY_VERIFICATION_KEY,
-                EventType.KEY_VERIFICATION_MAC,
-                EventType.KEY_VERIFICATION_CANCEL,
-                EventType.KEY_VERIFICATION_DONE,
-                EventType.KEY_VERIFICATION_READY,
-                EventType.MESSAGE,
-                EventType.ENCRYPTED)
-        )
-    }
-
-    override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {
-        // Should we ignore when it's an initial sync?
-        val events = changeSet.insertions
-                .asSequence()
-                .mapNotNull { results[it]?.asDomain() }
-                .filterNot {
-                    // ignore local echos
-                    LocalEcho.isLocalEchoId(it.eventId ?: "")
-                }
-                .toList()
-
-        roomVerificationUpdateTask.configureWith(
-                RoomVerificationUpdateTask.Params(events, verificationService, cryptoService)
-        ).executeBy(taskExecutor)
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/verification/VerificationMessageProcessor.kt

@@ -0,0 +1,168 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package im.vector.matrix.android.internal.crypto.verification
+
+import im.vector.matrix.android.api.session.crypto.CryptoService
+import im.vector.matrix.android.api.session.crypto.MXCryptoError
+import im.vector.matrix.android.api.session.crypto.verification.VerificationService
+import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.LocalEcho
+import im.vector.matrix.android.api.session.events.model.toModel
+import im.vector.matrix.android.api.session.room.model.message.MessageContent
+import im.vector.matrix.android.api.session.room.model.message.MessageRelationContent
+import im.vector.matrix.android.api.session.room.model.message.MessageType
+import im.vector.matrix.android.api.session.room.model.message.MessageVerificationReadyContent
+import im.vector.matrix.android.api.session.room.model.message.MessageVerificationRequestContent
+import im.vector.matrix.android.api.session.room.model.message.MessageVerificationStartContent
+import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
+import im.vector.matrix.android.internal.database.model.EventInsertType
+import im.vector.matrix.android.internal.di.DeviceId
+import im.vector.matrix.android.internal.di.UserId
+import im.vector.matrix.android.internal.session.EventInsertLiveProcessor
+import io.realm.Realm
+import timber.log.Timber
+import java.util.ArrayList
+import javax.inject.Inject
+
+internal class VerificationMessageProcessor @Inject constructor(
+        private val cryptoService: CryptoService,
+        private val verificationService: DefaultVerificationService,
+        @UserId private val userId: String,
+        @DeviceId private val deviceId: String?
+) : EventInsertLiveProcessor {
+
+    private val transactionsHandledByOtherDevice = ArrayList<String>()
+
+    private val allowedTypes = listOf(
+            EventType.KEY_VERIFICATION_START,
+            EventType.KEY_VERIFICATION_ACCEPT,
+            EventType.KEY_VERIFICATION_KEY,
+            EventType.KEY_VERIFICATION_MAC,
+            EventType.KEY_VERIFICATION_CANCEL,
+            EventType.KEY_VERIFICATION_DONE,
+            EventType.KEY_VERIFICATION_READY,
+            EventType.MESSAGE,
+            EventType.ENCRYPTED
+    )
+
+    override fun shouldProcess(eventId: String, eventType: String, insertType: EventInsertType): Boolean {
+        if (insertType != EventInsertType.INCREMENTAL_SYNC) {
+            return false
+        }
+        return allowedTypes.contains(eventType) && !LocalEcho.isLocalEchoId(eventId)
+    }
+
+    override suspend fun process(realm: Realm, event: Event) {
+        Timber.v("## SAS Verification live observer: received msgId: ${event.eventId} msgtype: ${event.type} from ${event.senderId}")
+
+        // If the request is in the future by more than 5 minutes or more than 10 minutes in the past,
+        // the message should be ignored by the receiver.
+
+        if (!VerificationService.isValidRequest(event.ageLocalTs
+                        ?: event.originServerTs)) return Unit.also {
+            Timber.d("## SAS Verification live observer: msgId: ${event.eventId} is outdated")
+        }
+
+        // decrypt if needed?
+        if (event.isEncrypted() && event.mxDecryptionResult == null) {
+            // TODO use a global event decryptor? attache to session and that listen to new sessionId?
+            // for now decrypt sync
+            try {
+                val result = cryptoService.decryptEvent(event, "")
+                event.mxDecryptionResult = OlmDecryptionResult(
+                        payload = result.clearEvent,
+                        senderKey = result.senderCurve25519Key,
+                        keysClaimed = result.claimedEd25519Key?.let { mapOf("ed25519" to it) },
+                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
+                )
+            } catch (e: MXCryptoError) {
+                Timber.e("## SAS Failed to decrypt event: ${event.eventId}")
+                verificationService.onPotentiallyInterestingEventRoomFailToDecrypt(event)
+            }
+        }
+        Timber.v("## SAS Verification live observer: received msgId: ${event.eventId} type: ${event.getClearType()}")
+
+        // Relates to is not encrypted
+        val relatesToEventId = event.content.toModel<MessageRelationContent>()?.relatesTo?.eventId
+
+        if (event.senderId == userId) {
+            // If it's send from me, we need to keep track of Requests or Start
+            // done from another device of mine
+
+            if (EventType.MESSAGE == event.getClearType()) {
+                val msgType = event.getClearContent().toModel<MessageContent>()?.msgType
+                if (MessageType.MSGTYPE_VERIFICATION_REQUEST == msgType) {
+                    event.getClearContent().toModel<MessageVerificationRequestContent>()?.let {
+                        if (it.fromDevice != deviceId) {
+                            // The verification is requested from another device
+                            Timber.v("## SAS Verification live observer: Transaction requested from other device  tid:${event.eventId} ")
+                            event.eventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
+                        }
+                    }
+                }
+            } else if (EventType.KEY_VERIFICATION_START == event.getClearType()) {
+                event.getClearContent().toModel<MessageVerificationStartContent>()?.let {
+                    if (it.fromDevice != deviceId) {
+                        // The verification is started from another device
+                        Timber.v("## SAS Verification live observer: Transaction started by other device  tid:$relatesToEventId ")
+                        relatesToEventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
+                        verificationService.onRoomRequestHandledByOtherDevice(event)
+                    }
+                }
+            } else if (EventType.KEY_VERIFICATION_READY == event.getClearType()) {
+                event.getClearContent().toModel<MessageVerificationReadyContent>()?.let {
+                    if (it.fromDevice != deviceId) {
+                        // The verification is started from another device
+                        Timber.v("## SAS Verification live observer: Transaction started by other device  tid:$relatesToEventId ")
+                        relatesToEventId?.let { txId -> transactionsHandledByOtherDevice.add(txId) }
+                        verificationService.onRoomRequestHandledByOtherDevice(event)
+                    }
+                }
+            } else if (EventType.KEY_VERIFICATION_CANCEL == event.getClearType() || EventType.KEY_VERIFICATION_DONE == event.getClearType()) {
+                relatesToEventId?.let {
+                    transactionsHandledByOtherDevice.remove(it)
+                    verificationService.onRoomRequestHandledByOtherDevice(event)
+                }
+            }
+
+            Timber.v("## SAS Verification ignoring message sent by me: ${event.eventId} type: ${event.getClearType()}")
+            return
+        }
+
+        if (relatesToEventId != null && transactionsHandledByOtherDevice.contains(relatesToEventId)) {
+            // Ignore this event, it is directed to another of my devices
+            Timber.v("## SAS Verification live observer: Ignore Transaction handled by other device  tid:$relatesToEventId ")
+            return
+        }
+        when (event.getClearType()) {
+            EventType.KEY_VERIFICATION_START,
+            EventType.KEY_VERIFICATION_ACCEPT,
+            EventType.KEY_VERIFICATION_KEY,
+            EventType.KEY_VERIFICATION_MAC,
+            EventType.KEY_VERIFICATION_CANCEL,
+            EventType.KEY_VERIFICATION_READY,
+            EventType.KEY_VERIFICATION_DONE -> {
+                verificationService.onRoomEvent(event)
+            }
+            EventType.MESSAGE               -> {
+                if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.msgType) {
+                    verificationService.onRoomRequestReceived(event)
+                }
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/DatabaseCleaner.kt

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.database
+
+import im.vector.matrix.android.internal.database.helper.nextDisplayIndex
+import im.vector.matrix.android.internal.database.model.ChunkEntity
+import im.vector.matrix.android.internal.database.model.ChunkEntityFields
+import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields
+import im.vector.matrix.android.internal.di.SessionDatabase
+import im.vector.matrix.android.internal.session.SessionLifecycleObserver
+import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
+import im.vector.matrix.android.internal.task.TaskExecutor
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import timber.log.Timber
+import javax.inject.Inject
+
+private const val MAX_NUMBER_OF_EVENTS_IN_DB = 35_000L
+private const val MIN_NUMBER_OF_EVENTS_BY_CHUNK = 300
+
+/**
+ * This class makes sure to stay under a maximum number of events as it makes Realm to be unusable when listening to events
+ * when the database is getting too big. This will try incrementally to remove the biggest chunks until we get below the threshold.
+ * We make sure to still have a minimum number of events so it's not becoming unusable.
+ * So this won't work for users with a big number of very active rooms.
+ */
+internal class DatabaseCleaner @Inject constructor(@SessionDatabase private val realmConfiguration: RealmConfiguration,
+                                                   private val taskExecutor: TaskExecutor) : SessionLifecycleObserver {
+
+    override fun onStart() {
+        taskExecutor.executorScope.launch(Dispatchers.Default) {
+            awaitTransaction(realmConfiguration) { realm ->
+                val allRooms = realm.where(RoomEntity::class.java).findAll()
+                Timber.v("There are ${allRooms.size} rooms in this session")
+                cleanUp(realm, MAX_NUMBER_OF_EVENTS_IN_DB / 2L)
+            }
+        }
+    }
+
+    private suspend fun cleanUp(realm: Realm, threshold: Long) {
+        val numberOfEvents = realm.where(EventEntity::class.java).findAll().size
+        val numberOfTimelineEvents = realm.where(TimelineEventEntity::class.java).findAll().size
+        Timber.v("Number of events in db: $numberOfEvents | Number of timeline events in db: $numberOfTimelineEvents")
+        if (threshold <= MIN_NUMBER_OF_EVENTS_BY_CHUNK || numberOfTimelineEvents < MAX_NUMBER_OF_EVENTS_IN_DB) {
+            Timber.v("Db is low enough")
+        } else {
+            val thresholdChunks = realm.where(ChunkEntity::class.java)
+                    .greaterThan(ChunkEntityFields.NUMBER_OF_TIMELINE_EVENTS, threshold)
+                    .findAll()
+
+            Timber.v("There are ${thresholdChunks.size} chunks to clean with more than $threshold events")
+            for (chunk in thresholdChunks) {
+                val maxDisplayIndex = chunk.nextDisplayIndex(PaginationDirection.FORWARDS)
+                val thresholdDisplayIndex = maxDisplayIndex - threshold
+                val eventsToRemove = chunk.timelineEvents.where().lessThan(TimelineEventEntityFields.DISPLAY_INDEX, thresholdDisplayIndex).findAll()
+                Timber.v("There are ${eventsToRemove.size} events to clean in chunk: ${chunk.identifier()} from room ${chunk.room?.first()?.roomId}")
+                chunk.numberOfTimelineEvents = chunk.numberOfTimelineEvents - eventsToRemove.size
+                eventsToRemove.forEach {
+                    val canDeleteRoot = it.root?.stateKey == null
+                    if (canDeleteRoot) {
+                        it.root?.deleteFromRealm()
+                    }
+                    it.readReceipts?.readReceipts?.deleteAllFromRealm()
+                    it.readReceipts?.deleteFromRealm()
+                    it.annotations?.apply {
+                        editSummary?.deleteFromRealm()
+                        pollResponseSummary?.deleteFromRealm()
+                        referencesSummaryEntity?.deleteFromRealm()
+                        reactionsSummary.deleteAllFromRealm()
+                    }
+                    it.annotations?.deleteFromRealm()
+                    it.readReceipts?.deleteFromRealm()
+                    it.deleteFromRealm()
+                }
+                // We reset the prevToken so we will need to fetch again.
+                chunk.prevToken = null
+            }
+            cleanUp(realm, (threshold / 1.5).toLong())
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/EventInsertLiveObserver.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.database
+
+import com.zhuinden.monarchy.Monarchy
+import im.vector.matrix.android.api.session.crypto.CryptoService
+import im.vector.matrix.android.api.session.crypto.MXCryptoError
+import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
+import im.vector.matrix.android.internal.database.mapper.asDomain
+import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.EventInsertEntity
+import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.di.SessionDatabase
+import im.vector.matrix.android.internal.session.EventInsertLiveProcessor
+import io.realm.RealmConfiguration
+import io.realm.RealmResults
+import kotlinx.coroutines.launch
+import timber.log.Timber
+import javax.inject.Inject
+
+internal class EventInsertLiveObserver @Inject constructor(@SessionDatabase realmConfiguration: RealmConfiguration,
+                                                           private val processors: Set<@JvmSuppressWildcards EventInsertLiveProcessor>,
+                                                           private val cryptoService: CryptoService)
+    : RealmLiveEntityObserver<EventInsertEntity>(realmConfiguration) {
+
+    override val query = Monarchy.Query<EventInsertEntity> {
+        it.where(EventInsertEntity::class.java)
+    }
+
+    override fun onChange(results: RealmResults<EventInsertEntity>) {
+        if (!results.isLoaded || results.isEmpty()) {
+            return
+        }
+        Timber.v("EventInsertEntity updated with ${results.size} results in db")
+        val filteredEvents = results.mapNotNull {
+            if (shouldProcess(it)) {
+                results.realm.copyFromRealm(it)
+            } else {
+                null
+            }
+        }
+        Timber.v("There are ${filteredEvents.size} events to process")
+        observerScope.launch {
+            awaitTransaction(realmConfiguration) { realm ->
+                filteredEvents.forEach { eventInsert ->
+                    val eventId = eventInsert.eventId
+                    val event = EventEntity.where(realm, eventId).findFirst()
+                    if (event == null) {
+                        Timber.v("Event $eventId not found")
+                        return@forEach
+                    }
+                    val domainEvent = event.asDomain()
+                    decryptIfNeeded(domainEvent)
+                    processors.filter {
+                        it.shouldProcess(eventId, domainEvent.getClearType(), eventInsert.insertType)
+                    }.forEach {
+                        it.process(realm, domainEvent)
+                    }
+                }
+                realm.where(EventInsertEntity::class.java).findAll().deleteAllFromRealm()
+            }
+        }
+    }
+
+    private fun decryptIfNeeded(event: Event) {
+        if (event.isEncrypted() && event.mxDecryptionResult == null) {
+            try {
+                val result = cryptoService.decryptEvent(event, event.roomId ?: "")
+                event.mxDecryptionResult = OlmDecryptionResult(
+                        payload = result.clearEvent,
+                        senderKey = result.senderCurve25519Key,
+                        keysClaimed = result.claimedEd25519Key?.let { k -> mapOf("ed25519" to k) },
+                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
+                )
+            } catch (e: MXCryptoError) {
+                Timber.v("Call service: Failed to decrypt event")
+                // TODO -> we should keep track of this and retry, or aggregation will be broken
+            }
+        }
+    }
+
+    private fun shouldProcess(eventInsertEntity: EventInsertEntity): Boolean {
+        return processors.any {
+            it.shouldProcess(eventInsertEntity.eventId, eventInsertEntity.eventType, eventInsertEntity.insertType)
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/RealmLiveEntityObserver.kt

@@ -19,8 +19,8 @@ package im.vector.matrix.android.internal.database
 import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.internal.session.SessionLifecycleObserver
 import im.vector.matrix.android.internal.util.createBackgroundHandler
-import io.realm.OrderedRealmCollectionChangeListener
 import io.realm.Realm
+import io.realm.RealmChangeListener
 import io.realm.RealmConfiguration
 import io.realm.RealmObject
 import io.realm.RealmResults
@@ -30,10 +30,10 @@ import kotlinx.coroutines.cancelChildren
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 
-internal interface LiveEntityObserver: SessionLifecycleObserver
+internal interface LiveEntityObserver : SessionLifecycleObserver
 
 internal abstract class RealmLiveEntityObserver<T : RealmObject>(protected val realmConfiguration: RealmConfiguration)
-    : LiveEntityObserver, OrderedRealmCollectionChangeListener<RealmResults<T>> {
+    : LiveEntityObserver, RealmChangeListener<RealmResults<T>> {
 
     private companion object {
         val BACKGROUND_HANDLER = createBackgroundHandler("LIVE_ENTITY_BACKGROUND")

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt

@@ -115,6 +115,7 @@ internal fun ChunkEntity.addTimelineEvent(roomId: String,
             true
         }
     }
+    numberOfTimelineEvents++
     timelineEvents.add(timelineEventEntity)
 }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/EventMapper.kt

@@ -45,6 +45,11 @@ internal object EventMapper {
         eventEntity.redacts = event.redacts
         eventEntity.age = event.unsignedData?.age ?: event.originServerTs
         eventEntity.unsignedData = uds
+        eventEntity.decryptionResultJson = event.mxDecryptionResult?.let {
+            MoshiProvider.providesMoshi().adapter<OlmDecryptionResult>(OlmDecryptionResult::class.java).toJson(it)
+        }
+        eventEntity.decryptionErrorReason = event.mCryptoErrorReason
+        eventEntity.decryptionErrorCode = event.mCryptoError?.name
         return eventEntity
     }
 
@@ -85,6 +90,7 @@ internal object EventMapper {
             it.mCryptoError = eventEntity.decryptionErrorCode?.let { errorCode ->
                 MXCryptoError.ErrorType.valueOf(errorCode)
             }
+            it.mCryptoErrorReason = eventEntity.decryptionErrorReason
         }
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/GroupMapper.kt

@@ -1,34 +0,0 @@
-/*
- * Copyright 2019 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.database.mapper
-
-import im.vector.matrix.android.api.session.group.Group
-import im.vector.matrix.android.internal.database.model.GroupEntity
-import im.vector.matrix.android.internal.session.group.DefaultGroup
-
-internal object GroupMapper {
-
-    fun map(groupEntity: GroupEntity): Group {
-        return DefaultGroup(
-                groupEntity.groupId
-        )
-    }
-}
-
-internal fun GroupEntity.asDomain(): Group {
-    return GroupMapper.map(this)
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/RoomSummaryMapper.kt

@@ -19,9 +19,11 @@ package im.vector.matrix.android.internal.database.mapper
 import im.vector.matrix.android.api.session.room.model.RoomSummary
 import im.vector.matrix.android.api.session.room.model.tag.RoomTag
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
+import im.vector.matrix.android.internal.session.typing.DefaultTypingUsersTracker
 import javax.inject.Inject
 
-internal class RoomSummaryMapper @Inject constructor(private val timelineEventMapper: TimelineEventMapper) {
+internal class RoomSummaryMapper @Inject constructor(private val timelineEventMapper: TimelineEventMapper,
+                                                     private val typingUsersTracker: DefaultTypingUsersTracker) {
 
     fun map(roomSummaryEntity: RoomSummaryEntity): RoomSummary {
         val tags = roomSummaryEntity.tags.map {
@@ -31,10 +33,13 @@ internal class RoomSummaryMapper @Inject constructor(private val timelineEventMa
         val latestEvent = roomSummaryEntity.latestPreviewableEvent?.let {
             timelineEventMapper.map(it, buildReadReceipts = false)
         }
+        // typings are updated through the sync where room summary entity gets updated no matter what, so it's ok get there
+        val typingUsers = typingUsersTracker.getTypingUsers(roomSummaryEntity.roomId)
 
         return RoomSummary(
                 roomId = roomSummaryEntity.roomId,
                 displayName = roomSummaryEntity.displayName ?: "",
+                name = roomSummaryEntity.name ?: "",
                 topic = roomSummaryEntity.topic ?: "",
                 avatarUrl = roomSummaryEntity.avatarUrl ?: "",
                 isDirect = roomSummaryEntity.isDirect,
@@ -46,6 +51,7 @@ internal class RoomSummaryMapper @Inject constructor(private val timelineEventMa
                 notificationCount = roomSummaryEntity.notificationCount,
                 hasUnreadMessages = roomSummaryEntity.hasUnreadMessages,
                 tags = tags,
+                typingUsers = typingUsers,
                 membership = roomSummaryEntity.membership,
                 versioningState = roomSummaryEntity.versioningState,
                 readMarkerId = roomSummaryEntity.readMarkerId,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt

@@ -27,6 +27,7 @@ internal open class ChunkEntity(@Index var prevToken: String? = null,
                                 @Index var nextToken: String? = null,
                                 var stateEvents: RealmList<EventEntity> = RealmList(),
                                 var timelineEvents: RealmList<TimelineEventEntity> = RealmList(),
+                                var numberOfTimelineEvents: Long = 0,
                                 // Only one chunk will have isLastForward == true
                                 @Index var isLastForward: Boolean = false,
                                 @Index var isLastBackward: Boolean = false

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/EventEntity.kt

@@ -37,6 +37,7 @@ internal open class EventEntity(@Index var eventId: String = "",
                                 var redacts: String? = null,
                                 var decryptionResultJson: String? = null,
                                 var decryptionErrorCode: String? = null,
+                                var decryptionErrorReason: String? = null,
                                 var ageLocalTs: Long? = null
 ) : RealmObject() {
 
@@ -62,5 +63,6 @@ internal open class EventEntity(@Index var eventId: String = "",
         val adapter = MoshiProvider.providesMoshi().adapter<OlmDecryptionResult>(OlmDecryptionResult::class.java)
         decryptionResultJson = adapter.toJson(decryptionResult)
         decryptionErrorCode = null
+        decryptionErrorReason = null
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/EventInsertEntity.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.database.model
+
+import io.realm.RealmObject
+
+/**
+ * This class is used to get notification on new events being inserted. It's to avoid realm getting slow when listening to insert
+ * in EventEntity table.
+ */
+internal open class EventInsertEntity(var eventId: String = "",
+                                      var eventType: String = ""
+) : RealmObject() {
+
+    private var insertTypeStr: String = EventInsertType.INCREMENTAL_SYNC.name
+    var insertType: EventInsertType
+        get() {
+            return EventInsertType.valueOf(insertTypeStr)
+        }
+        set(value) {
+            insertTypeStr = value.name
+        }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/EventInsertType.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.database.model;
+
+public enum EventInsertType {
+    INITIAL_SYNC,
+    INCREMENTAL_SYNC,
+    PAGINATION,
+    LOCAL_ECHO
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/GroupEntity.kt

@@ -22,8 +22,7 @@ import io.realm.annotations.PrimaryKey
 
 /**
  * This class is used to store group info (groupId and membership) from the sync response.
- * Then [im.vector.matrix.android.internal.session.group.GroupSummaryUpdater] observes change and
- * makes requests to fetch group information from the homeserver
+ * Then GetGroupDataTask is called regularly to fetch group information from the homeserver.
  */
 internal open class GroupEntity(@PrimaryKey var groupId: String = "")
     : RealmObject() {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/RoomSummaryEntity.kt

@@ -28,6 +28,7 @@ internal open class RoomSummaryEntity(
         @PrimaryKey var roomId: String = "",
         var displayName: String? = "",
         var avatarUrl: String? = "",
+        var name: String? = "",
         var topic: String? = "",
         var latestPreviewableEvent: TimelineEventEntity? = null,
         var heroes: RealmList<String> = RealmList(),

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/SessionRealmModule.kt

@@ -25,6 +25,7 @@ import io.realm.annotations.RealmModule
         classes = [
             ChunkEntity::class,
             EventEntity::class,
+            EventInsertEntity::class,
             TimelineEventEntity::class,
             FilterEntity::class,
             GroupEntity::class,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/EventEntityQueries.kt

@@ -18,16 +18,28 @@ package im.vector.matrix.android.internal.database.query
 
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
+import im.vector.matrix.android.internal.database.model.EventInsertEntity
+import im.vector.matrix.android.internal.database.model.EventInsertType
 import io.realm.Realm
 import io.realm.RealmList
 import io.realm.RealmQuery
 import io.realm.kotlin.where
 
-internal fun EventEntity.copyToRealmOrIgnore(realm: Realm): EventEntity {
-    return realm.where<EventEntity>()
-        .equalTo(EventEntityFields.EVENT_ID, eventId)
-        .equalTo(EventEntityFields.ROOM_ID, roomId)
-        .findFirst() ?: realm.copyToRealm(this)
+internal fun EventEntity.copyToRealmOrIgnore(realm: Realm, insertType: EventInsertType): EventEntity {
+    val eventEntity = realm.where<EventEntity>()
+            .equalTo(EventEntityFields.EVENT_ID, eventId)
+            .equalTo(EventEntityFields.ROOM_ID, roomId)
+            .findFirst()
+    return if (eventEntity == null) {
+        val insertEntity = EventInsertEntity(eventId = eventId, eventType = type).apply {
+            this.insertType = insertType
+        }
+        realm.insert(insertEntity)
+        // copy this event entity and return it
+        realm.copyToRealm(this)
+    } else {
+        eventEntity
+    }
 }
 
 internal fun EventEntity.Companion.where(realm: Realm, eventId: String): RealmQuery<EventEntity> {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/GroupEntityQueries.kt

@@ -19,6 +19,7 @@ package im.vector.matrix.android.internal.database.query
 import im.vector.matrix.android.api.session.room.model.Membership
 import im.vector.matrix.android.internal.database.model.GroupEntity
 import im.vector.matrix.android.internal.database.model.GroupEntityFields
+import im.vector.matrix.android.internal.query.process
 import io.realm.Realm
 import io.realm.RealmQuery
 import io.realm.kotlin.where
@@ -28,10 +29,6 @@ internal fun GroupEntity.Companion.where(realm: Realm, groupId: String): RealmQu
             .equalTo(GroupEntityFields.GROUP_ID, groupId)
 }
 
-internal fun GroupEntity.Companion.where(realm: Realm, membership: Membership? = null): RealmQuery<GroupEntity> {
-    val query = realm.where<GroupEntity>()
-    if (membership != null) {
-        query.equalTo(GroupEntityFields.MEMBERSHIP_STR, membership.name)
-    }
-    return query
+internal fun GroupEntity.Companion.where(realm: Realm, memberships: List<Membership>): RealmQuery<GroupEntity> {
+    return realm.where<GroupEntity>().process(GroupEntityFields.MEMBERSHIP_STR, memberships)
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/GroupSummaryEntityQueries.kt

@@ -20,6 +20,7 @@ import im.vector.matrix.android.internal.database.model.GroupSummaryEntity
 import im.vector.matrix.android.internal.database.model.GroupSummaryEntityFields
 import io.realm.Realm
 import io.realm.RealmQuery
+import io.realm.kotlin.createObject
 import io.realm.kotlin.where
 
 internal fun GroupSummaryEntity.Companion.where(realm: Realm, groupId: String? = null): RealmQuery<GroupSummaryEntity> {
@@ -34,3 +35,7 @@ internal fun GroupSummaryEntity.Companion.where(realm: Realm, groupIds: List<Str
     return realm.where<GroupSummaryEntity>()
             .`in`(GroupSummaryEntityFields.GROUP_ID, groupIds.toTypedArray())
 }
+
+internal fun GroupSummaryEntity.Companion.getOrCreate(realm: Realm, groupId: String): GroupSummaryEntity {
+    return where(realm, groupId).findFirst() ?: realm.createObject(groupId)
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/AuthQualifiers.kt

@@ -29,3 +29,11 @@ internal annotation class AuthenticatedIdentity
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)
 internal annotation class Unauthenticated
+
+@Qualifier
+@Retention(AnnotationRetention.RUNTIME)
+internal annotation class UnauthenticatedWithCertificate
+
+@Qualifier
+@Retention(AnnotationRetention.RUNTIME)
+internal annotation class UnauthenticatedWithCertificateWithProgress

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/FileQualifiers.kt

@@ -24,7 +24,7 @@ internal annotation class SessionFilesDirectory
 
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)
-internal annotation class SessionCacheDirectory
+internal annotation class SessionDownloadsDirectory
 
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/MatrixComponent.kt

@@ -27,6 +27,8 @@ import im.vector.matrix.android.api.auth.AuthenticationService
 import im.vector.matrix.android.internal.SessionManager
 import im.vector.matrix.android.internal.auth.AuthModule
 import im.vector.matrix.android.internal.auth.SessionParamsStore
+import im.vector.matrix.android.internal.session.MockHttpInterceptor
+import im.vector.matrix.android.internal.session.TestInterceptor
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.util.BackgroundDetectionObserver
 import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers
@@ -34,7 +36,7 @@ import okhttp3.OkHttpClient
 import org.matrix.olm.OlmManager
 import java.io.File
 
-@Component(modules = [MatrixModule::class, NetworkModule::class, AuthModule::class])
+@Component(modules = [MatrixModule::class, NetworkModule::class, AuthModule::class, NoOpTestModule::class])
 @MatrixScope
 internal interface MatrixComponent {
 
@@ -45,6 +47,9 @@ internal interface MatrixComponent {
     @Unauthenticated
     fun okHttpClient(): OkHttpClient
 
+    @MockHttpInterceptor
+    fun testInterceptor(): TestInterceptor?
+
     fun authenticationService(): AuthenticationService
 
     fun context(): Context

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/NoOpTestModule.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.di
+
+import dagger.Module
+import dagger.Provides
+import im.vector.matrix.android.internal.session.MockHttpInterceptor
+import im.vector.matrix.android.internal.session.TestInterceptor
+
+@Module
+internal object NoOpTestModule {
+
+    @Provides
+    @JvmStatic
+    @MockHttpInterceptor
+    fun providesTestInterceptor(): TestInterceptor? = null
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/WorkManagerProvider.kt

@@ -21,7 +21,9 @@ import androidx.work.Constraints
 import androidx.work.ListenableWorker
 import androidx.work.NetworkType
 import androidx.work.OneTimeWorkRequestBuilder
+import androidx.work.PeriodicWorkRequestBuilder
 import androidx.work.WorkManager
+import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 
 internal class WorkManagerProvider @Inject constructor(
@@ -39,6 +41,14 @@ internal class WorkManagerProvider @Inject constructor(
             OneTimeWorkRequestBuilder<W>()
                     .addTag(tag)
 
+    /**
+     * Create a PeriodicWorkRequestBuilder, with the Matrix SDK tag
+     */
+    inline fun <reified W : ListenableWorker> matrixPeriodicWorkRequestBuilder(repeatInterval: Long,
+                                                                               repeatIntervalTimeUnit: TimeUnit) =
+            PeriodicWorkRequestBuilder<W>(repeatInterval, repeatIntervalTimeUnit)
+                    .addTag(tag)
+
     /**
      * Cancel all works instantiated by the Matrix SDK for the current session, and not those from the SDK client, or for other sessions
      */

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/riot/CertUtil.java

@@ -1,284 +0,0 @@
-/*
- * Copyright (c) 2020 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.legacy.riot;
-
-import android.util.Pair;
-
-import androidx.annotation.NonNull;
-
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-
-import okhttp3.CipherSuite;
-import okhttp3.ConnectionSpec;
-import okhttp3.TlsVersion;
-import timber.log.Timber;
-
-/*
- * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
- */
-
-/**
- * Various utility classes for dealing with X509Certificates
- */
-public class CertUtil {
-    /**
-     * Generates the SHA-256 fingerprint of the given certificate
-     *
-     * @param cert the certificate.
-     * @return the finger print
-     * @throws CertificateException the certificate exception
-     */
-    public static byte[] generateSha256Fingerprint(X509Certificate cert) throws CertificateException {
-        return generateFingerprint(cert, "SHA-256");
-    }
-
-    /**
-     * Generates the SHA-1 fingerprint of the given certificate
-     *
-     * @param cert the certificated
-     * @return the SHA1 fingerprint
-     * @throws CertificateException the certificate exception
-     */
-    public static byte[] generateSha1Fingerprint(X509Certificate cert) throws CertificateException {
-        return generateFingerprint(cert, "SHA-1");
-    }
-
-    /**
-     * Generate the fingerprint for a dedicated type.
-     *
-     * @param cert the certificate
-     * @param type the type
-     * @return the fingerprint
-     * @throws CertificateException certificate exception
-     */
-    private static byte[] generateFingerprint(X509Certificate cert, String type) throws CertificateException {
-        final byte[] fingerprint;
-        final MessageDigest md;
-        try {
-            md = MessageDigest.getInstance(type);
-        } catch (Exception e) {
-            // This really *really* shouldn't throw, as java should always have a SHA-256 and SHA-1 impl.
-            throw new CertificateException(e);
-        }
-
-        fingerprint = md.digest(cert.getEncoded());
-
-        return fingerprint;
-    }
-
-    final private static char[] hexArray = "0123456789ABCDEF".toCharArray();
-
-    /**
-     * Convert the fingerprint to an hexa string.
-     *
-     * @param fingerprint the fingerprint
-     * @return the hexa string.
-     */
-    public static String fingerprintToHexString(byte[] fingerprint) {
-        return fingerprintToHexString(fingerprint, ' ');
-    }
-
-    public static String fingerprintToHexString(byte[] fingerprint, char sep) {
-        char[] hexChars = new char[fingerprint.length * 3];
-        for (int j = 0; j < fingerprint.length; j++) {
-            int v = fingerprint[j] & 0xFF;
-            hexChars[j * 3] = hexArray[v >>> 4];
-            hexChars[j * 3 + 1] = hexArray[v & 0x0F];
-            hexChars[j * 3 + 2] = sep;
-        }
-        return new String(hexChars, 0, hexChars.length - 1);
-    }
-
-    /**
-     * Recursively checks the exception to see if it was caused by an
-     * UnrecognizedCertificateException
-     *
-     * @param e the throwable.
-     * @return The UnrecognizedCertificateException if exists, else null.
-     */
-    public static UnrecognizedCertificateException getCertificateException(Throwable e) {
-        int i = 0; // Just in case there is a getCause loop
-        while (e != null && i < 10) {
-            if (e instanceof UnrecognizedCertificateException) {
-                return (UnrecognizedCertificateException) e;
-            }
-            e = e.getCause();
-            i++;
-        }
-
-        return null;
-    }
-
-    /**
-     * Create a SSLSocket factory for a HS config.
-     *
-     * @param hsConfig the HS config.
-     * @return SSLSocket factory
-     */
-    public static Pair<SSLSocketFactory, X509TrustManager> newPinnedSSLSocketFactory(HomeServerConnectionConfig hsConfig) {
-        X509TrustManager defaultTrustManager = null;
-
-        // If we haven't specified that we wanted to pin the certs, fallback to standard
-        // X509 checks if fingerprints don't match.
-        if (!hsConfig.shouldPin()) {
-            TrustManagerFactory trustManagerFactory = null;
-
-            // get the PKIX instance
-            try {
-                trustManagerFactory = TrustManagerFactory.getInstance("PKIX");
-            } catch (NoSuchAlgorithmException e) {
-                Timber.e(e, "## newPinnedSSLSocketFactory() : TrustManagerFactory.getInstance failed");
-            }
-
-            // it doesn't exist, use the default one.
-            if (trustManagerFactory == null) {
-                try {
-                    trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-                } catch (NoSuchAlgorithmException e) {
-                    Timber.e(e, "## newPinnedSSLSocketFactory() : TrustManagerFactory.getInstance with default algorithm failed");
-                }
-            }
-
-            if (trustManagerFactory != null) {
-                try {
-                    trustManagerFactory.init((KeyStore) null);
-                    TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
-
-                    for (int i = 0; i < trustManagers.length; i++) {
-                        if (trustManagers[i] instanceof X509TrustManager) {
-                            defaultTrustManager = (X509TrustManager) trustManagers[i];
-                            break;
-                        }
-                    }
-                } catch (KeyStoreException e) {
-                    Timber.e(e, "## newPinnedSSLSocketFactory()");
-                }
-            }
-        }
-
-        X509TrustManager trustManager = new PinnedTrustManager(hsConfig.getAllowedFingerprints(), defaultTrustManager);
-
-        TrustManager[] trustManagers = new TrustManager[]{
-                trustManager
-        };
-
-        SSLSocketFactory sslSocketFactory;
-
-        try {
-            if (hsConfig.forceUsageOfTlsVersions() && hsConfig.getAcceptedTlsVersions() != null) {
-                // Force usage of accepted Tls Versions for Android < 20
-                sslSocketFactory = new TLSSocketFactory(trustManagers, hsConfig.getAcceptedTlsVersions());
-            } else {
-                SSLContext sslContext = SSLContext.getInstance("TLS");
-                sslContext.init(null, trustManagers, new java.security.SecureRandom());
-                sslSocketFactory = sslContext.getSocketFactory();
-            }
-        } catch (Exception e) {
-            // This is too fatal
-            throw new RuntimeException(e);
-        }
-
-        return new Pair<>(sslSocketFactory, trustManager);
-    }
-
-    /**
-     * Create a Host name verifier for a hs config.
-     *
-     * @param hsConfig the hs config.
-     * @return a new HostnameVerifier.
-     */
-    public static HostnameVerifier newHostnameVerifier(HomeServerConnectionConfig hsConfig) {
-        final HostnameVerifier defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
-        final List<Fingerprint> trusted_fingerprints = hsConfig.getAllowedFingerprints();
-
-        return new HostnameVerifier() {
-            @Override
-            public boolean verify(String hostname, SSLSession session) {
-                if (defaultVerifier.verify(hostname, session)) return true;
-                if (trusted_fingerprints == null || trusted_fingerprints.size() == 0) return false;
-
-                // If remote cert matches an allowed fingerprint, just accept it.
-                try {
-                    for (Certificate cert : session.getPeerCertificates()) {
-                        for (Fingerprint allowedFingerprint : trusted_fingerprints) {
-                            if (allowedFingerprint != null && cert instanceof X509Certificate && allowedFingerprint.matchesCert((X509Certificate) cert)) {
-                                return true;
-                            }
-                        }
-                    }
-                } catch (SSLPeerUnverifiedException e) {
-                    return false;
-                } catch (CertificateException e) {
-                    return false;
-                }
-
-                return false;
-            }
-        };
-    }
-
-    /**
-     * Create a list of accepted TLS specifications for a hs config.
-     *
-     * @param hsConfig the hs config.
-     * @param url      the url of the end point, used to check if we have to enable CLEARTEXT communication.
-     * @return a list of accepted TLS specifications.
-     */
-    public static List<ConnectionSpec> newConnectionSpecs(@NonNull HomeServerConnectionConfig hsConfig, @NonNull String url) {
-        final ConnectionSpec.Builder builder = new ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS);
-
-        final List<TlsVersion> tlsVersions = hsConfig.getAcceptedTlsVersions();
-        if (null != tlsVersions) {
-            builder.tlsVersions(tlsVersions.toArray(new TlsVersion[0]));
-        }
-
-        final List<CipherSuite> tlsCipherSuites = hsConfig.getAcceptedTlsCipherSuites();
-        if (null != tlsCipherSuites) {
-            builder.cipherSuites(tlsCipherSuites.toArray(new CipherSuite[0]));
-        }
-
-        builder.supportsTlsExtensions(hsConfig.shouldAcceptTlsExtensions());
-
-        List<ConnectionSpec> list = new ArrayList<>();
-
-        list.add(builder.build());
-
-        if (url.startsWith("http://")) {
-            list.add(ConnectionSpec.CLEARTEXT);
-        }
-
-        return list;
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/riot/Fingerprint.java

@@ -21,8 +21,6 @@ import android.util.Base64;
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
 import java.util.Arrays;
 
 /*
@@ -40,20 +38,10 @@ public class Fingerprint {
 
     private final HashType mHashType;
     private final byte[] mBytes;
-    private String mDisplayableHexRepr;
 
     public Fingerprint(HashType hashType, byte[] bytes) {
         mHashType = hashType;
         mBytes = bytes;
-        mDisplayableHexRepr = null;
-    }
-
-    public static Fingerprint newSha256Fingerprint(X509Certificate cert) throws CertificateException {
-        return new Fingerprint(HashType.SHA256, CertUtil.generateSha256Fingerprint(cert));
-    }
-
-    public static Fingerprint newSha1Fingerprint(X509Certificate cert) throws CertificateException {
-        return new Fingerprint(HashType.SHA1, CertUtil.generateSha1Fingerprint(cert));
     }
 
     public HashType getType() {
@@ -64,14 +52,6 @@ public class Fingerprint {
         return mBytes;
     }
 
-    public String getBytesAsHexString() {
-        if (mDisplayableHexRepr == null) {
-            mDisplayableHexRepr = CertUtil.fingerprintToHexString(mBytes);
-        }
-
-        return mDisplayableHexRepr;
-    }
-
     public JSONObject toJson() throws JSONException {
         JSONObject obj = new JSONObject();
         obj.put("bytes", Base64.encodeToString(getBytes(), Base64.DEFAULT));
@@ -95,24 +75,6 @@ public class Fingerprint {
         return new Fingerprint(hashType, fingerprintBytes);
     }
 
-    public boolean matchesCert(X509Certificate cert) throws CertificateException {
-        Fingerprint o = null;
-        switch (mHashType) {
-            case SHA256:
-                o = Fingerprint.newSha256Fingerprint(cert);
-                break;
-            case SHA1:
-                o = Fingerprint.newSha1Fingerprint(cert);
-                break;
-        }
-
-        return equals(o);
-    }
-
-    public String toString() {
-        return String.format("Fingerprint{type: '%s', fingeprint: '%s'}", mHashType.toString(), getBytesAsHexString());
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/riot/PinnedTrustManager.java

@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2020 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.legacy.riot;
-
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.List;
-
-import javax.annotation.Nullable;
-import javax.net.ssl.X509TrustManager;
-
-/*
- * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
- */
-
-/**
- * Implements a TrustManager that checks Certificates against an explicit list of known
- * fingerprints.
- */
-public class PinnedTrustManager implements X509TrustManager {
-    private final List<Fingerprint> mFingerprints;
-    @Nullable
-    private final X509TrustManager mDefaultTrustManager;
-
-    /**
-     * @param fingerprints        An array of SHA256 cert fingerprints
-     * @param defaultTrustManager Optional trust manager to fall back on if cert does not match
-     *                            any of the fingerprints. Can be null.
-     */
-    public PinnedTrustManager(List<Fingerprint> fingerprints, @Nullable X509TrustManager defaultTrustManager) {
-        mFingerprints = fingerprints;
-        mDefaultTrustManager = defaultTrustManager;
-    }
-
-    @Override
-    public void checkClientTrusted(X509Certificate[] chain, String s) throws CertificateException {
-        try {
-            if (mDefaultTrustManager != null) {
-                mDefaultTrustManager.checkClientTrusted(
-                        chain, s
-                );
-                return;
-            }
-        } catch (CertificateException e) {
-            // If there is an exception we fall back to checking fingerprints
-            if (mFingerprints == null || mFingerprints.size() == 0) {
-                throw new UnrecognizedCertificateException(chain[0], Fingerprint.newSha256Fingerprint(chain[0]), e.getCause());
-            }
-        }
-        checkTrusted("client", chain);
-    }
-
-    @Override
-    public void checkServerTrusted(X509Certificate[] chain, String s) throws CertificateException {
-        try {
-            if (mDefaultTrustManager != null) {
-                mDefaultTrustManager.checkServerTrusted(
-                        chain, s
-                );
-                return;
-            }
-        } catch (CertificateException e) {
-            // If there is an exception we fall back to checking fingerprints
-            if (mFingerprints == null || mFingerprints.isEmpty()) {
-                throw new UnrecognizedCertificateException(chain[0], Fingerprint.newSha256Fingerprint(chain[0]), e.getCause());
-            }
-        }
-        checkTrusted("server", chain);
-    }
-
-    private void checkTrusted(String type, X509Certificate[] chain) throws CertificateException {
-        X509Certificate cert = chain[0];
-
-        boolean found = false;
-        if (mFingerprints != null) {
-            for (Fingerprint allowedFingerprint : mFingerprints) {
-                if (allowedFingerprint != null && allowedFingerprint.matchesCert(cert)) {
-                    found = true;
-                    break;
-                }
-            }
-        }
-
-        if (!found) {
-            throw new UnrecognizedCertificateException(cert, Fingerprint.newSha256Fingerprint(cert), null);
-        }
-    }
-
-    @Override
-    public X509Certificate[] getAcceptedIssuers() {
-        return new X509Certificate[0];
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/riot/TLSSocketFactory.java

@@ -1,135 +0,0 @@
-/*
- * Copyright (c) 2020 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.legacy.riot;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.UnknownHostException;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocket;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-
-import okhttp3.TlsVersion;
-import timber.log.Timber;
-
-/*
- * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
- */
-
-/**
- * Force the usage of Tls versions on every created socket
- * Inspired from https://blog.dev-area.net/2015/08/13/android-4-1-enable-tls-1-1-and-tls-1-2/
- */
-/*package*/ class TLSSocketFactory extends SSLSocketFactory {
-    private SSLSocketFactory internalSSLSocketFactory;
-
-    private String[] enabledProtocols;
-
-    /**
-     * Constructor
-     *
-     * @param trustPinned
-     * @param acceptedTlsVersions
-     * @throws KeyManagementException
-     * @throws NoSuchAlgorithmException
-     */
-    /*package*/ TLSSocketFactory(TrustManager[] trustPinned, List<TlsVersion> acceptedTlsVersions) throws KeyManagementException, NoSuchAlgorithmException {
-        SSLContext context = SSLContext.getInstance("TLS");
-        context.init(null, trustPinned, new SecureRandom());
-        internalSSLSocketFactory = context.getSocketFactory();
-
-        enabledProtocols = new String[acceptedTlsVersions.size()];
-        int i = 0;
-        for (TlsVersion tlsVersion : acceptedTlsVersions) {
-            enabledProtocols[i] = tlsVersion.javaName();
-            i++;
-        }
-    }
-
-    @Override
-    public String[] getDefaultCipherSuites() {
-        return internalSSLSocketFactory.getDefaultCipherSuites();
-    }
-
-    @Override
-    public String[] getSupportedCipherSuites() {
-        return internalSSLSocketFactory.getSupportedCipherSuites();
-    }
-
-    @Override
-    public Socket createSocket() throws IOException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket());
-    }
-
-    @Override
-    public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket(s, host, port, autoClose));
-    }
-
-    @Override
-    public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port));
-    }
-
-    @Override
-    public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException, UnknownHostException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port, localHost, localPort));
-    }
-
-    @Override
-    public Socket createSocket(InetAddress host, int port) throws IOException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port));
-    }
-
-    @Override
-    public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
-        return enableTLSOnSocket(internalSSLSocketFactory.createSocket(address, port, localAddress, localPort));
-    }
-
-    private Socket enableTLSOnSocket(Socket socket) {
-        if (socket != null && (socket instanceof SSLSocket)) {
-            SSLSocket sslSocket = (SSLSocket) socket;
-
-            List<String> supportedProtocols = Arrays.asList(sslSocket.getSupportedProtocols());
-            List<String> filteredEnabledProtocols = new ArrayList<>();
-
-            for (String protocol : enabledProtocols) {
-                if (supportedProtocols.contains(protocol)) {
-                    filteredEnabledProtocols.add(protocol);
-                }
-            }
-
-            if (!filteredEnabledProtocols.isEmpty()) {
-                try {
-                    sslSocket.setEnabledProtocols(filteredEnabledProtocols.toArray(new String[filteredEnabledProtocols.size()]));
-                } catch (Exception e) {
-                    Timber.e(e, "Exception");
-                }
-            }
-        }
-        return socket;
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/riot/UnrecognizedCertificateException.java

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2020 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.legacy.riot;
-
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-
-/*
- * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
- */
-
-/**
- * Thrown when we are given a certificate that does match the certificate we were told to
- * expect.
- */
-public class UnrecognizedCertificateException extends CertificateException {
-    private final X509Certificate mCert;
-    private final Fingerprint mFingerprint;
-
-    public UnrecognizedCertificateException(X509Certificate cert, Fingerprint fingerprint, Throwable cause) {
-        super("Unrecognized certificate with unknown fingerprint: " + cert.getSubjectDN(), cause);
-        mCert = cert;
-        mFingerprint = fingerprint;
-    }
-
-    public X509Certificate getCertificate() {
-        return mCert;
-    }
-
-    public Fingerprint getFingerprint() {
-        return mFingerprint;
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/Request.kt

@@ -18,6 +18,7 @@ package im.vector.matrix.android.internal.network
 
 import im.vector.matrix.android.api.failure.Failure
 import im.vector.matrix.android.api.failure.shouldBeRetried
+import im.vector.matrix.android.internal.network.ssl.CertUtil
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.delay
 import org.greenrobot.eventbus.EventBus
@@ -26,7 +27,7 @@ import retrofit2.awaitResponse
 import java.io.IOException
 
 internal suspend inline fun <DATA : Any> executeRequest(eventBus: EventBus?,
-                                                  block: Request<DATA>.() -> Unit) = Request<DATA>(eventBus).apply(block).execute()
+                                                        block: Request<DATA>.() -> Unit) = Request<DATA>(eventBus).apply(block).execute()
 
 internal class Request<DATA : Any>(private val eventBus: EventBus?) {
 
@@ -48,6 +49,15 @@ internal class Request<DATA : Any>(private val eventBus: EventBus?) {
                 throw response.toFailure(eventBus)
             }
         } catch (exception: Throwable) {
+            // Check if this is a certificateException
+            CertUtil.getCertificateException(exception)
+                    // TODO Support certificate error once logged
+                    // ?.also { unrecognizedCertificateException ->
+                    //    // Send the error to the bus, for a global management
+                    //    eventBus?.post(GlobalError.CertificateError(unrecognizedCertificateException))
+                    // }
+                    ?.also { unrecognizedCertificateException -> throw unrecognizedCertificateException }
+
             if (isRetryable && currentRetryCount++ < maxRetryCount && exception.shouldBeRetried()) {
                 delay(currentDelay)
                 currentDelay = (currentDelay * 2L).coerceAtMost(maxDelay)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/RetrofitFactory.kt

@@ -26,7 +26,19 @@ import retrofit2.Retrofit
 import retrofit2.converter.moshi.MoshiConverterFactory
 import javax.inject.Inject
 
-class RetrofitFactory @Inject constructor(private val moshi: Moshi) {
+internal class RetrofitFactory @Inject constructor(private val moshi: Moshi) {
+
+    /**
+     * Use only for authentication service
+     */
+    fun create(okHttpClient: OkHttpClient, baseUrl: String): Retrofit {
+        return Retrofit.Builder()
+                .baseUrl(baseUrl.ensureTrailingSlash())
+                .client(okHttpClient)
+                .addConverterFactory(UnitConverterFactory)
+                .addConverterFactory(MoshiConverterFactory.create(moshi))
+                .build()
+    }
 
     fun create(okHttpClient: Lazy<OkHttpClient>, baseUrl: String): Retrofit {
         return Retrofit.Builder()

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/httpclient/OkHttpClientUtil.kt

@@ -16,24 +16,38 @@
 
 package im.vector.matrix.android.internal.network.httpclient
 
+import im.vector.matrix.android.api.auth.data.HomeServerConnectionConfig
 import im.vector.matrix.android.internal.network.AccessTokenInterceptor
 import im.vector.matrix.android.internal.network.interceptors.CurlLoggingInterceptor
+import im.vector.matrix.android.internal.network.ssl.CertUtil
 import im.vector.matrix.android.internal.network.token.AccessTokenProvider
 import okhttp3.OkHttpClient
+import timber.log.Timber
 
-internal fun OkHttpClient.addAccessTokenInterceptor(accessTokenProvider: AccessTokenProvider): OkHttpClient {
-    return newBuilder()
-            .apply {
-                // Remove the previous CurlLoggingInterceptor, to add it after the accessTokenInterceptor
-                val existingCurlInterceptors = interceptors().filterIsInstance<CurlLoggingInterceptor>()
-                interceptors().removeAll(existingCurlInterceptors)
+internal fun OkHttpClient.Builder.addAccessTokenInterceptor(accessTokenProvider: AccessTokenProvider): OkHttpClient.Builder {
+    // Remove the previous CurlLoggingInterceptor, to add it after the accessTokenInterceptor
+    val existingCurlInterceptors = interceptors().filterIsInstance<CurlLoggingInterceptor>()
+    interceptors().removeAll(existingCurlInterceptors)
 
-                addInterceptor(AccessTokenInterceptor(accessTokenProvider))
+    addInterceptor(AccessTokenInterceptor(accessTokenProvider))
 
-                // Re add eventually the curl logging interceptors
-                existingCurlInterceptors.forEach {
-                    addInterceptor(it)
-                }
-            }
-            .build()
+    // Re add eventually the curl logging interceptors
+    existingCurlInterceptors.forEach {
+        addInterceptor(it)
+    }
+
+    return this
+}
+
+internal fun OkHttpClient.Builder.addSocketFactory(homeServerConnectionConfig: HomeServerConnectionConfig): OkHttpClient.Builder {
+    try {
+        val pair = CertUtil.newPinnedSSLSocketFactory(homeServerConnectionConfig)
+        sslSocketFactory(pair.sslSocketFactory, pair.x509TrustManager)
+        hostnameVerifier(CertUtil.newHostnameVerifier(homeServerConnectionConfig))
+        connectionSpecs(CertUtil.newConnectionSpecs(homeServerConnectionConfig))
+    } catch (e: Exception) {
+        Timber.e(e, "addSocketFactory failed")
+    }
+
+    return this
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/ssl/CertUtil.kt

@@ -16,29 +16,30 @@
 
 package im.vector.matrix.android.internal.network.ssl
 
-import android.util.Pair
 import im.vector.matrix.android.api.auth.data.HomeServerConnectionConfig
 import okhttp3.ConnectionSpec
+import okhttp3.internal.tls.OkHostnameVerifier
 import timber.log.Timber
 import java.security.KeyStore
 import java.security.MessageDigest
 import java.security.cert.CertificateException
 import java.security.cert.X509Certificate
 import javax.net.ssl.HostnameVerifier
-import javax.net.ssl.HttpsURLConnection
 import javax.net.ssl.SSLContext
 import javax.net.ssl.SSLPeerUnverifiedException
 import javax.net.ssl.SSLSocketFactory
 import javax.net.ssl.TrustManager
 import javax.net.ssl.TrustManagerFactory
 import javax.net.ssl.X509TrustManager
-import kotlin.experimental.and
 
 /**
  * Various utility classes for dealing with X509Certificates
  */
 internal object CertUtil {
 
+    // Set to false to do some test
+    private const val USE_DEFAULT_HOSTNAME_VERIFIER = true
+
     private val hexArray = "0123456789ABCDEF".toCharArray()
 
     /**
@@ -95,11 +96,10 @@ internal object CertUtil {
      * @param fingerprint the fingerprint
      * @return the hexa string.
      */
-    @JvmOverloads
     fun fingerprintToHexString(fingerprint: ByteArray, sep: Char = ' '): String {
         val hexChars = CharArray(fingerprint.size * 3)
         for (j in fingerprint.indices) {
-            val v = (fingerprint[j] and 0xFF.toByte()).toInt()
+            val v = (fingerprint[j].toInt() and 0xFF)
             hexChars[j * 3] = hexArray[v.ushr(4)]
             hexChars[j * 3 + 1] = hexArray[v and 0x0F]
             hexChars[j * 3 + 2] = sep
@@ -128,13 +128,18 @@ internal object CertUtil {
         return null
     }
 
+    internal data class PinnedSSLSocketFactory(
+            val sslSocketFactory: SSLSocketFactory,
+            val x509TrustManager: X509TrustManager
+    )
+
     /**
      * Create a SSLSocket factory for a HS config.
      *
      * @param hsConfig the HS config.
      * @return SSLSocket factory
      */
-    fun newPinnedSSLSocketFactory(hsConfig: HomeServerConnectionConfig): Pair<SSLSocketFactory, X509TrustManager> {
+    fun newPinnedSSLSocketFactory(hsConfig: HomeServerConnectionConfig): PinnedSSLSocketFactory {
         try {
             var defaultTrustManager: X509TrustManager? = null
 
@@ -155,7 +160,7 @@ internal object CertUtil {
                     try {
                         tf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
                     } catch (e: Exception) {
-                        Timber.e(e, "## addRule : onBingRuleUpdateFailure failed")
+                        Timber.e(e, "## newPinnedSSLSocketFactory() : TrustManagerFactory.getInstance of default failed")
                     }
                 }
 
@@ -170,7 +175,7 @@ internal object CertUtil {
                 }
             }
 
-            val trustPinned = arrayOf<TrustManager>(PinnedTrustManager(hsConfig.allowedFingerprints, defaultTrustManager))
+            val trustPinned = arrayOf<TrustManager>(PinnedTrustManagerProvider.provide(hsConfig.allowedFingerprints, defaultTrustManager))
 
             val sslSocketFactory: SSLSocketFactory
 
@@ -183,7 +188,7 @@ internal object CertUtil {
                 sslSocketFactory = sslContext.socketFactory
             }
 
-            return Pair<SSLSocketFactory, X509TrustManager>(sslSocketFactory, defaultTrustManager)
+            return PinnedSSLSocketFactory(sslSocketFactory, defaultTrustManager!!)
         } catch (e: Exception) {
             throw RuntimeException(e)
         }
@@ -196,11 +201,14 @@ internal object CertUtil {
      * @return a new HostnameVerifier.
      */
     fun newHostnameVerifier(hsConfig: HomeServerConnectionConfig): HostnameVerifier {
-        val defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier()
+        val defaultVerifier: HostnameVerifier = OkHostnameVerifier // HttpsURLConnection.getDefaultHostnameVerifier()
         val trustedFingerprints = hsConfig.allowedFingerprints
 
         return HostnameVerifier { hostname, session ->
-            if (defaultVerifier.verify(hostname, session)) return@HostnameVerifier true
+            if (USE_DEFAULT_HOSTNAME_VERIFIER) {
+                if (defaultVerifier.verify(hostname, session)) return@HostnameVerifier true
+            }
+            // TODO How to recover from this error?
             if (trustedFingerprints.isEmpty()) return@HostnameVerifier false
 
             // If remote cert matches an allowed fingerprint, just accept it.
@@ -231,12 +239,12 @@ internal object CertUtil {
     fun newConnectionSpecs(hsConfig: HomeServerConnectionConfig): List<ConnectionSpec> {
         val builder = ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
         val tlsVersions = hsConfig.tlsVersions
-        if (null != tlsVersions) {
+        if (null != tlsVersions && tlsVersions.isNotEmpty()) {
             builder.tlsVersions(*tlsVersions.toTypedArray())
         }
 
         val tlsCipherSuites = hsConfig.tlsCipherSuites
-        if (null != tlsCipherSuites) {
+        if (null != tlsCipherSuites && tlsCipherSuites.isNotEmpty()) {
             builder.cipherSuites(*tlsCipherSuites.toTypedArray())
         }
 
@@ -244,7 +252,8 @@ internal object CertUtil {
         builder.supportsTlsExtensions(hsConfig.shouldAcceptTlsExtensions)
         val list = ArrayList<ConnectionSpec>()
         list.add(builder.build())
-        if (hsConfig.allowHttpExtension) {
+        // TODO: we should display a warning if user enter an http url
+        if (hsConfig.allowHttpExtension || hsConfig.homeServerUri.toString().startsWith("http://")) {
             list.add(ConnectionSpec.CLEARTEXT)
         }
         return list