Merge pull request #945 from vector-im/feature/stabilization_2

Feature/stabilization 2

.idea/codeStyles/Project.xml

@@ -1,5 +1,6 @@
 <component name="ProjectCodeStyleConfiguration">
   <code_scheme name="Project" version="173">
+    <option name="RIGHT_MARGIN" value="160" />
     <AndroidXmlCodeStyleSettings>
       <option name="ARRANGEMENT_SETTINGS_MIGRATED_TO_191" value="true" />
     </AndroidXmlCodeStyleSettings>

CHANGES.md

@@ -5,7 +5,8 @@ Features ✨:
  -
 
 Improvements 🙌:
- -
+ - Improve navigation to the timeline (#789, #862)
+ - Improve network detection. It is now based on the sync request status (#873, #882) 
 
 Other changes:
  -

build.gradle

@@ -47,23 +47,11 @@ allprojects {
         jcenter()
     }
 
-    tasks.withType(JavaCompile).all {
-        options.compilerArgs += [
-                '-Adagger.gradle.incremental=enabled'
-        ]
-    }
-
     tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
         // Warnings are potential errors, so stop ignoring them
         kotlinOptions.allWarningsAsErrors = true
     }
 
-    afterEvaluate {
-        extensions.findByName("kapt")?.arguments {
-            arg("dagger.gradle.incremental", "enabled")
-        }
-    }
-
 }
 
 task clean(type: Delete) {

matrix-sdk-android-rx/src/main/java/im/vector/matrix/rx/LiveDataObservable.kt

@@ -20,6 +20,7 @@ import androidx.lifecycle.LiveData
 import androidx.lifecycle.Observer
 import io.reactivex.Observable
 import io.reactivex.android.MainThreadDisposable
+import io.reactivex.android.schedulers.AndroidSchedulers
 import io.reactivex.schedulers.Schedulers
 
 private class LiveDataObservable<T>(
@@ -57,6 +58,14 @@ private class LiveDataObservable<T>(
     }
 }
 
-fun <T> LiveData<T>.asObservable(): Observable<T> {
+internal fun <T> LiveData<T>.asObservable(): Observable<T> {
     return LiveDataObservable(this).observeOn(Schedulers.computation())
 }
+
+internal fun <T> Observable<T>.startWithCallable(supplier: () -> T): Observable<T> {
+    val startObservable = Observable
+            .fromCallable(supplier)
+            .subscribeOn(Schedulers.io())
+            .observeOn(AndroidSchedulers.mainThread())
+    return startWith(startObservable)
+}

matrix-sdk-android-rx/src/main/java/im/vector/matrix/rx/RxRoom.kt

@@ -41,7 +41,9 @@ class RxRoom(private val room: Room, private val session: Session) {
     fun liveRoomSummary(): Observable<Optional<RoomSummary>> {
         val summaryObservable = room.getRoomSummaryLive()
                 .asObservable()
-                .startWith(room.roomSummary().toOptional())
+                .startWithCallable {
+                    room.roomSummary().toOptional()
+                }
                 .doOnNext { Timber.v("RX: summary emitted for: ${it.getOrNull()?.roomId}") }
 
         val memberIdsChangeObservable = summaryObservable
@@ -97,7 +99,9 @@ class RxRoom(private val room: Room, private val session: Session) {
 
     fun liveRoomMembers(queryParams: RoomMemberQueryParams): Observable<List<RoomMemberSummary>> {
         val roomMembersObservable = room.getRoomMembersLive(queryParams).asObservable()
-                .startWith(room.getRoomMembers(queryParams))
+                .startWithCallable {
+                    room.getRoomMembers(queryParams)
+                }
                 .doOnNext { Timber.v("RX: room members emitted. Size: ${it.size}") }
 
         // TODO Do it only for room members of the room (switchMap)
@@ -127,17 +131,23 @@ class RxRoom(private val room: Room, private val session: Session) {
 
     fun liveAnnotationSummary(eventId: String): Observable<Optional<EventAnnotationsSummary>> {
         return room.getEventAnnotationsSummaryLive(eventId).asObservable()
-                .startWith(room.getEventAnnotationsSummary(eventId).toOptional())
+                .startWithCallable {
+                    room.getEventAnnotationsSummary(eventId).toOptional()
+                }
     }
 
     fun liveTimelineEvent(eventId: String): Observable<Optional<TimelineEvent>> {
         return room.getTimeLineEventLive(eventId).asObservable()
-                .startWith(room.getTimeLineEvent(eventId).toOptional())
+                .startWithCallable {
+                    room.getTimeLineEvent(eventId).toOptional()
+                }
     }
 
-    fun liveStateEvent(eventType: String): Observable<Optional<Event>> {
-        return room.getStateEventLive(eventType).asObservable()
-                .startWith(room.getStateEvent(eventType).toOptional())
+    fun liveStateEvent(eventType: String, stateKey: String): Observable<Optional<Event>> {
+        return room.getStateEventLive(eventType, stateKey).asObservable()
+                .startWithCallable {
+                    room.getStateEvent(eventType, stateKey).toOptional()
+                }
     }
 
     fun liveReadMarker(): Observable<Optional<String>> {

matrix-sdk-android-rx/src/main/java/im/vector/matrix/rx/RxSession.kt

@@ -40,7 +40,9 @@ class RxSession(private val session: Session) {
 
     fun liveRoomSummaries(queryParams: RoomSummaryQueryParams): Observable<List<RoomSummary>> {
         val summariesObservable = session.getRoomSummariesLive(queryParams).asObservable()
-                .startWith(session.getRoomSummaries(queryParams))
+                .startWithCallable {
+                    session.getRoomSummaries(queryParams)
+                }
                 .doOnNext { Timber.v("RX: summaries emitted: size: ${it.size}") }
 
         val cryptoDeviceInfoObservable = session.getLiveCryptoDeviceInfo().asObservable()
@@ -69,12 +71,16 @@ class RxSession(private val session: Session) {
 
     fun liveGroupSummaries(queryParams: GroupSummaryQueryParams): Observable<List<GroupSummary>> {
         return session.getGroupSummariesLive(queryParams).asObservable()
-                .startWith(session.getGroupSummaries(queryParams))
+                .startWithCallable {
+                    session.getGroupSummaries(queryParams)
+                }
     }
 
     fun liveBreadcrumbs(): Observable<List<RoomSummary>> {
         return session.getBreadcrumbsLive().asObservable()
-                .startWith(session.getBreadcrumbs())
+                .startWithCallable {
+                    session.getBreadcrumbs()
+                }
     }
 
     fun liveSyncState(): Observable<SyncState> {
@@ -87,7 +93,9 @@ class RxSession(private val session: Session) {
 
     fun liveUser(userId: String): Observable<Optional<User>> {
         return session.getUserLive(userId).asObservable()
-                .startWith(session.getUser(userId).toOptional())
+                .startWithCallable {
+                    session.getUser(userId).toOptional()
+                }
     }
 
     fun liveUsers(): Observable<List<User>> {

matrix-sdk-android/build.gradle

@@ -92,10 +92,11 @@ dependencies {
 
     def arrow_version = "0.8.2"
     def moshi_version = '1.8.0'
-    def lifecycle_version = '2.1.0'
+    def lifecycle_version = '2.2.0'
+    def arch_version = '2.1.0'
     def coroutines_version = "1.3.2"
     def markwon_version = '3.1.0'
-    def daggerVersion = '2.24'
+    def daggerVersion = '2.25.4'
 
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version"
@@ -104,14 +105,13 @@ dependencies {
     implementation "androidx.appcompat:appcompat:1.1.0"
 
     implementation "androidx.lifecycle:lifecycle-extensions:$lifecycle_version"
-    kapt "androidx.lifecycle:lifecycle-compiler:$lifecycle_version"
+    implementation "androidx.lifecycle:lifecycle-common-java8:$lifecycle_version"
 
     // Network
     implementation 'com.squareup.retrofit2:retrofit:2.6.2'
     implementation 'com.squareup.retrofit2:converter-moshi:2.6.2'
     implementation 'com.squareup.okhttp3:okhttp:4.2.2'
     implementation 'com.squareup.okhttp3:logging-interceptor:4.2.2'
-    implementation 'com.novoda:merlin:1.2.0'
     implementation "com.squareup.moshi:moshi-adapters:$moshi_version"
     kapt "com.squareup.moshi:moshi-kotlin-codegen:$moshi_version"
 
@@ -125,7 +125,7 @@ dependencies {
     kapt 'dk.ilios:realmfieldnameshelper:1.1.1'
 
     // Work
-    implementation "androidx.work:work-runtime-ktx:2.3.0-beta02"
+    implementation "androidx.work:work-runtime-ktx:2.3.0"
 
     // FP
     implementation "io.arrow-kt:arrow-core:$arrow_version"
@@ -167,7 +167,7 @@ dependencies {
     androidTestImplementation 'org.amshove.kluent:kluent-android:1.44'
     // Note: version sticks to 1.9.2 due to https://github.com/mockk/mockk/issues/281
     androidTestImplementation 'io.mockk:mockk-android:1.9.2.kotlin12'
-    androidTestImplementation "androidx.arch.core:core-testing:$lifecycle_version"
+    androidTestImplementation "androidx.arch.core:core-testing:$arch_version"
     androidTestImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
 
 }

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/CommonTestHelper.kt

@@ -117,6 +117,10 @@ class CommonTestHelper(context: Context) {
             override fun onTimelineFailure(throwable: Throwable) {
             }
 
+            override fun onNewTimelineEvents(eventIds: List<String>) {
+                // noop
+            }
+
             override fun onTimelineUpdated(snapshot: List<TimelineEvent>) {
                 // TODO Count only new messages?
                 if (snapshot.count { it.root.type == EventType.MESSAGE } == nbOfMessages) {

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/common/CryptoTestHelper.kt

@@ -241,6 +241,11 @@ class CryptoTestHelper(val mTestHelper: CommonTestHelper) {
 
         val bobEventsListener = object : Timeline.Listener {
             override fun onTimelineFailure(throwable: Throwable) {
+                // noop
+            }
+
+            override fun onNewTimelineEvents(eventIds: List<String>) {
+                // noop
             }
 
             override fun onTimelineUpdated(snapshot: List<TimelineEvent>) {

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/session/room/timeline/ChunkEntityTest.kt

@@ -20,15 +20,15 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
 import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.InstrumentedTest
 import im.vector.matrix.android.api.session.events.model.Event
-import im.vector.matrix.android.internal.database.helper.add
-import im.vector.matrix.android.internal.database.helper.lastStateIndex
+import im.vector.matrix.android.api.session.room.send.SendState
+import im.vector.matrix.android.internal.database.helper.addTimelineEvent
 import im.vector.matrix.android.internal.database.helper.merge
+import im.vector.matrix.android.internal.database.mapper.toEntity
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.SessionRealmModule
 import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
 import im.vector.matrix.android.session.room.timeline.RoomDataHelper.createFakeListOfEvents
 import im.vector.matrix.android.session.room.timeline.RoomDataHelper.createFakeMessageEvent
-import im.vector.matrix.android.session.room.timeline.RoomDataHelper.createFakeRoomMemberEvent
 import io.realm.Realm
 import io.realm.RealmConfiguration
 import io.realm.kotlin.createObject
@@ -58,8 +58,11 @@ internal class ChunkEntityTest : InstrumentedTest {
     fun add_shouldAdd_whenNotAlreadyIncluded() {
         monarchy.runTransactionSync { realm ->
             val chunk: ChunkEntity = realm.createObject()
-            val fakeEvent = createFakeMessageEvent()
-            chunk.add("roomId", fakeEvent, PaginationDirection.FORWARDS)
+
+            val fakeEvent = createFakeMessageEvent().toEntity(ROOM_ID, SendState.SYNCED).let {
+                realm.copyToRealmOrUpdate(it)
+            }
+            chunk.addTimelineEvent(ROOM_ID, fakeEvent, PaginationDirection.FORWARDS, emptyMap())
             chunk.timelineEvents.size shouldEqual 1
         }
     }
@@ -68,65 +71,23 @@ internal class ChunkEntityTest : InstrumentedTest {
     fun add_shouldNotAdd_whenAlreadyIncluded() {
         monarchy.runTransactionSync { realm ->
             val chunk: ChunkEntity = realm.createObject()
-            val fakeEvent = createFakeMessageEvent()
-            chunk.add("roomId", fakeEvent, PaginationDirection.FORWARDS)
-            chunk.add("roomId", fakeEvent, PaginationDirection.FORWARDS)
+            val fakeEvent = createFakeMessageEvent().toEntity(ROOM_ID, SendState.SYNCED).let {
+                realm.copyToRealmOrUpdate(it)
+            }
+            chunk.addTimelineEvent(ROOM_ID, fakeEvent, PaginationDirection.FORWARDS, emptyMap())
+            chunk.addTimelineEvent(ROOM_ID, fakeEvent, PaginationDirection.FORWARDS, emptyMap())
             chunk.timelineEvents.size shouldEqual 1
         }
     }
 
-    @Test
-    fun add_shouldStateIndexIncremented_whenStateEventIsAddedForward() {
-        monarchy.runTransactionSync { realm ->
-            val chunk: ChunkEntity = realm.createObject()
-            val fakeEvent = createFakeRoomMemberEvent()
-            chunk.add("roomId", fakeEvent, PaginationDirection.FORWARDS)
-            chunk.lastStateIndex(PaginationDirection.FORWARDS) shouldEqual 1
-        }
-    }
-
-    @Test
-    fun add_shouldStateIndexNotIncremented_whenNoStateEventIsAdded() {
-        monarchy.runTransactionSync { realm ->
-            val chunk: ChunkEntity = realm.createObject()
-            val fakeEvent = createFakeMessageEvent()
-            chunk.add("roomId", fakeEvent, PaginationDirection.FORWARDS)
-            chunk.lastStateIndex(PaginationDirection.FORWARDS) shouldEqual 0
-        }
-    }
-
-    @Test
-    fun addAll_shouldStateIndexIncremented_whenStateEventsAreAddedForward() {
-        monarchy.runTransactionSync { realm ->
-            val chunk: ChunkEntity = realm.createObject()
-            val fakeEvents = createFakeListOfEvents(30)
-            val numberOfStateEvents = fakeEvents.filter { it.isStateEvent() }.size
-            chunk.addAll("roomId", fakeEvents, PaginationDirection.FORWARDS)
-            chunk.lastStateIndex(PaginationDirection.FORWARDS) shouldEqual numberOfStateEvents
-        }
-    }
-
-    @Test
-    fun addAll_shouldStateIndexDecremented_whenStateEventsAreAddedBackward() {
-        monarchy.runTransactionSync { realm ->
-            val chunk: ChunkEntity = realm.createObject()
-            val fakeEvents = createFakeListOfEvents(30)
-            val numberOfStateEvents = fakeEvents.filter { it.isStateEvent() }.size
-            val lastIsState = fakeEvents.last().isStateEvent()
-            val expectedStateIndex = if (lastIsState) -numberOfStateEvents + 1 else -numberOfStateEvents
-            chunk.addAll("roomId", fakeEvents, PaginationDirection.BACKWARDS)
-            chunk.lastStateIndex(PaginationDirection.BACKWARDS) shouldEqual expectedStateIndex
-        }
-    }
-
     @Test
     fun merge_shouldAddEvents_whenMergingBackward() {
         monarchy.runTransactionSync { realm ->
             val chunk1: ChunkEntity = realm.createObject()
             val chunk2: ChunkEntity = realm.createObject()
-            chunk1.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk2.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk1.merge("roomId", chunk2, PaginationDirection.BACKWARDS)
+            chunk1.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk2.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk1.merge(ROOM_ID, chunk2, PaginationDirection.BACKWARDS)
             chunk1.timelineEvents.size shouldEqual 60
         }
     }
@@ -140,9 +101,9 @@ internal class ChunkEntityTest : InstrumentedTest {
             val eventsForChunk2 = eventsForChunk1 + createFakeListOfEvents(10)
             chunk1.isLastForward = true
             chunk2.isLastForward = false
-            chunk1.addAll("roomId", eventsForChunk1, PaginationDirection.FORWARDS)
-            chunk2.addAll("roomId", eventsForChunk2, PaginationDirection.BACKWARDS)
-            chunk1.merge("roomId", chunk2, PaginationDirection.BACKWARDS)
+            chunk1.addAll(ROOM_ID, eventsForChunk1, PaginationDirection.FORWARDS)
+            chunk2.addAll(ROOM_ID, eventsForChunk2, PaginationDirection.BACKWARDS)
+            chunk1.merge(ROOM_ID, chunk2, PaginationDirection.BACKWARDS)
             chunk1.timelineEvents.size shouldEqual 40
             chunk1.isLastForward.shouldBeTrue()
         }
@@ -155,9 +116,9 @@ internal class ChunkEntityTest : InstrumentedTest {
             val chunk2: ChunkEntity = realm.createObject()
             val prevToken = "prev_token"
             chunk1.prevToken = prevToken
-            chunk1.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk2.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk1.merge("roomId", chunk2, PaginationDirection.FORWARDS)
+            chunk1.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk2.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk1.merge(ROOM_ID, chunk2, PaginationDirection.FORWARDS)
             chunk1.prevToken shouldEqual prevToken
         }
     }
@@ -169,19 +130,25 @@ internal class ChunkEntityTest : InstrumentedTest {
             val chunk2: ChunkEntity = realm.createObject()
             val nextToken = "next_token"
             chunk1.nextToken = nextToken
-            chunk1.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk2.addAll("roomId", createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
-            chunk1.merge("roomId", chunk2, PaginationDirection.BACKWARDS)
+            chunk1.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk2.addAll(ROOM_ID, createFakeListOfEvents(30), PaginationDirection.BACKWARDS)
+            chunk1.merge(ROOM_ID, chunk2, PaginationDirection.BACKWARDS)
             chunk1.nextToken shouldEqual nextToken
         }
     }
 
     private fun ChunkEntity.addAll(roomId: String,
                                    events: List<Event>,
-                                   direction: PaginationDirection,
-                                   stateIndexOffset: Int = 0) {
+                                   direction: PaginationDirection) {
         events.forEach { event ->
-            add(roomId, event, direction, stateIndexOffset)
+            val fakeEvent = event.toEntity(roomId, SendState.SYNCED).let {
+                realm.copyToRealmOrUpdate(it)
+            }
+            addTimelineEvent(roomId, fakeEvent, direction, emptyMap())
         }
     }
+
+    companion object {
+        private const val ROOM_ID = "roomId"
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/MatrixCallback.kt

@@ -38,3 +38,8 @@ interface MatrixCallback<in T> {
         // no-op
     }
 }
+
+/**
+ * Basic no op implementation
+ */
+class NoOpMatrixCallback<T>: MatrixCallback<T>

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/pushrules/RoomMemberCountCondition.kt

@@ -59,7 +59,7 @@ class RoomMemberCountCondition(val iz: String) : Condition(Kind.room_member_coun
             val (prefix, count) = match.destructured
             return prefix to count.toInt()
         } catch (t: Throwable) {
-            Timber.d(t)
+            Timber.e(t, "Unable to parse 'is' field")
         }
         return null
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/Event.kt

@@ -200,7 +200,7 @@ data class Event(
 
 fun Event.isTextMessage(): Boolean {
     return getClearType() == EventType.MESSAGE
-            && when (getClearContent()?.toModel<MessageContent>()?.type) {
+            && when (getClearContent()?.toModel<MessageContent>()?.msgType) {
         MessageType.MSGTYPE_TEXT,
         MessageType.MSGTYPE_EMOTE,
         MessageType.MSGTYPE_NOTICE -> true
@@ -210,7 +210,7 @@ fun Event.isTextMessage(): Boolean {
 
 fun Event.isImageMessage(): Boolean {
     return getClearType() == EventType.MESSAGE
-            && when (getClearContent()?.toModel<MessageContent>()?.type) {
+            && when (getClearContent()?.toModel<MessageContent>()?.msgType) {
         MessageType.MSGTYPE_IMAGE -> true
         else                      -> false
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageAudioContent.kt

@@ -25,9 +25,9 @@ import im.vector.matrix.android.internal.crypto.model.rest.EncryptedFileInfo
 @JsonClass(generateAdapter = true)
 data class MessageAudioContent(
         /**
-         * Not documented
+         * Required. Must be 'm.audio'.
          */
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
 
         /**
          * Required. A description of the audio e.g. 'Bee Gees - Stayin' Alive', or some kind of content description for accessibility e.g. 'audio attachment'.
@@ -40,7 +40,7 @@ data class MessageAudioContent(
         @Json(name = "info") val audioInfo: AudioInfo? = null,
 
         /**
-         * Required. Required if the file is not encrypted. The URL (typically MXC URI) to the audio clip.
+         * Required if the file is not encrypted. The URL (typically MXC URI) to the audio clip.
          */
         @Json(name = "url") override val url: String? = null,
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageContent.kt

@@ -20,8 +20,7 @@ import im.vector.matrix.android.api.session.events.model.Content
 import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultContent
 
 interface MessageContent {
-    // TODO Rename to msgType
-    val type: String
+    val msgType: String
     val body: String
     val relatesTo: RelationDefaultContent?
     val newContent: Content?

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageDefaultContent.kt

@@ -23,7 +23,7 @@ import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultC
 
 @JsonClass(generateAdapter = true)
 data class MessageDefaultContent(
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
         @Json(name = "body") override val body: String,
         @Json(name = "m.relates_to") override val relatesTo: RelationDefaultContent? = null,
         @Json(name = "m.new_content") override val newContent: Content? = null

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageEmoteContent.kt

@@ -23,10 +23,26 @@ import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultC
 
 @JsonClass(generateAdapter = true)
 data class MessageEmoteContent(
-        @Json(name = "msgtype") override val type: String,
+        /**
+         * Required. Must be 'm.emote'.
+         */
+        @Json(name = "msgtype") override val msgType: String,
+
+        /**
+         * Required. The emote action to perform.
+         */
         @Json(name = "body") override val body: String,
+
+        /**
+         * The format used in the formatted_body. Currently only org.matrix.custom.html is supported.
+         */
         @Json(name = "format") val format: String? = null,
+
+        /**
+         * The formatted version of the body. This is required if format is specified.
+         */
         @Json(name = "formatted_body") val formattedBody: String? = null,
+
         @Json(name = "m.relates_to") override val relatesTo: RelationDefaultContent? = null,
         @Json(name = "m.new_content") override val newContent: Content? = null
 ) : MessageContent

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageEncryptedContent.kt

@@ -23,10 +23,13 @@ import im.vector.matrix.android.internal.crypto.model.rest.EncryptedFileInfo
  */
 interface MessageEncryptedContent : MessageContent {
     /**
-     * Required. Required if the file is unencrypted. The URL (typically MXC URI) to the image.
+     * Required if the file is unencrypted. The URL (typically MXC URI) to the image.
      */
     val url: String?
 
+    /**
+     * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
+     */
     val encryptedFileInfo: EncryptedFileInfo?
 }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageFileContent.kt

@@ -26,9 +26,9 @@ import im.vector.matrix.android.internal.crypto.model.rest.EncryptedFileInfo
 @JsonClass(generateAdapter = true)
 data class MessageFileContent(
         /**
-         * Not documented
+         * Required. Must be 'm.file'.
          */
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
 
         /**
          * Required. A human-readable description of the file. This is recommended to be the filename of the original upload.
@@ -46,13 +46,16 @@ data class MessageFileContent(
         @Json(name = "info") val info: FileInfo? = null,
 
         /**
-         * Required. Required if the file is unencrypted. The URL (typically MXC URI) to the file.
+         * Required if the file is unencrypted. The URL (typically MXC URI) to the file.
          */
         @Json(name = "url") override val url: String? = null,
 
         @Json(name = "m.relates_to") override val relatesTo: RelationDefaultContent? = null,
         @Json(name = "m.new_content") override val newContent: Content? = null,
 
+        /**
+         * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
+         */
         @Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
 ) : MessageEncryptedContent {
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageFormat.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.api.session.room.model.message
+
+object MessageFormat {
+    const val FORMAT_MATRIX_HTML = "org.matrix.custom.html"
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageImageContent.kt

@@ -27,7 +27,7 @@ data class MessageImageContent(
         /**
          * Required. Must be 'm.image'.
          */
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
 
         /**
          * Required. A textual representation of the image. This could be the alt text of the image, the filename of the image,
@@ -41,7 +41,7 @@ data class MessageImageContent(
         @Json(name = "info") override val info: ImageInfo? = null,
 
         /**
-         * Required. Required if the file is unencrypted. The URL (typically MXC URI) to the image.
+         * Required if the file is unencrypted. The URL (typically MXC URI) to the image.
          */
         @Json(name = "url") override val url: String? = null,
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageLocationContent.kt

@@ -24,9 +24,9 @@ import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultC
 @JsonClass(generateAdapter = true)
 data class MessageLocationContent(
         /**
-         * Not documented
+         * Required. Must be 'm.location'.
          */
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
 
         /**
          * Required. A description of the location e.g. 'Big Ben, London, UK', or some kind of content description for accessibility e.g. 'location attachment'.

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageNoticeContent.kt

@@ -23,10 +23,26 @@ import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultC
 
 @JsonClass(generateAdapter = true)
 data class MessageNoticeContent(
-        @Json(name = "msgtype") override val type: String,
+        /**
+         * Required. Must be 'm.notice'.
+         */
+        @Json(name = "msgtype") override val msgType: String,
+
+        /**
+         * Required. The notice text to send.
+         */
         @Json(name = "body") override val body: String,
+
+        /**
+         * The format used in the formatted_body. Currently only org.matrix.custom.html is supported.
+         */
         @Json(name = "format") val format: String? = null,
+
+        /**
+         * The formatted version of the body. This is required if format is specified.
+         */
         @Json(name = "formatted_body") val formattedBody: String? = null,
+
         @Json(name = "m.relates_to") override val relatesTo: RelationDefaultContent? = null,
         @Json(name = "m.new_content") override val newContent: Content? = null
 ) : MessageContent

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageStickerContent.kt

@@ -12,7 +12,6 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 package im.vector.matrix.android.api.session.room.model.message
@@ -28,7 +27,7 @@ data class MessageStickerContent(
         /**
          * Set in local, not from server
          */
-        override val type: String = MessageType.MSGTYPE_STICKER_LOCAL,
+        override val msgType: String = MessageType.MSGTYPE_STICKER_LOCAL,
 
         /**
          * Required. A textual representation of the image. This could be the alt text of the image, the filename of the image,
@@ -42,7 +41,7 @@ data class MessageStickerContent(
         @Json(name = "info") override val info: ImageInfo? = null,
 
         /**
-         * Required. Required if the file is unencrypted. The URL (typically MXC URI) to the image.
+         * Required if the file is unencrypted. The URL (typically MXC URI) to the image.
          */
         @Json(name = "url") override val url: String? = null,
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageTextContent.kt

@@ -23,10 +23,26 @@ import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultC
 
 @JsonClass(generateAdapter = true)
 data class MessageTextContent(
-        @Json(name = "msgtype") override val type: String,
+        /**
+         * Required. Must be 'm.text'.
+         */
+        @Json(name = "msgtype") override val msgType: String,
+
+        /**
+         * Required. The body of the message.
+         */
         @Json(name = "body") override val body: String,
+
+        /**
+         * The format used in the formatted_body. Currently only org.matrix.custom.html is supported.
+         */
         @Json(name = "format") val format: String? = null,
+
+        /**
+         * The formatted version of the body. This is required if format is specified.
+         */
         @Json(name = "formatted_body") val formattedBody: String? = null,
+
         @Json(name = "m.relates_to") override val relatesTo: RelationDefaultContent? = null,
         @Json(name = "m.new_content") override val newContent: Content? = null
 ) : MessageContent

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageType.kt

@@ -17,7 +17,6 @@
 package im.vector.matrix.android.api.session.room.model.message
 
 object MessageType {
-
     const val MSGTYPE_TEXT = "m.text"
     const val MSGTYPE_EMOTE = "m.emote"
     const val MSGTYPE_NOTICE = "m.notice"
@@ -27,7 +26,6 @@ object MessageType {
     const val MSGTYPE_LOCATION = "m.location"
     const val MSGTYPE_FILE = "m.file"
     const val MSGTYPE_VERIFICATION_REQUEST = "m.key.verification.request"
-    const val FORMAT_MATRIX_HTML = "org.matrix.custom.html"
     // Add, in local, a fake message type in order to StickerMessage can inherit Message class
     // Because sticker isn't a message type but a event type without msgtype field
     const val MSGTYPE_STICKER_LOCAL = "org.matrix.android.sdk.sticker"

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageVerificationRequestContent.kt

@@ -24,7 +24,7 @@ import im.vector.matrix.android.internal.crypto.verification.VerificationInfoReq
 
 @JsonClass(generateAdapter = true)
 data class MessageVerificationRequestContent(
-        @Json(name = "msgtype") override val type: String = MessageType.MSGTYPE_VERIFICATION_REQUEST,
+        @Json(name = "msgtype") override val msgType: String = MessageType.MSGTYPE_VERIFICATION_REQUEST,
         @Json(name = "body") override val body: String,
         @Json(name = "from_device") override val fromDevice: String?,
         @Json(name = "methods") override val methods: List<String>,

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/message/MessageVideoContent.kt

@@ -27,7 +27,7 @@ data class MessageVideoContent(
         /**
          * Required. Must be 'm.video'.
          */
-        @Json(name = "msgtype") override val type: String,
+        @Json(name = "msgtype") override val msgType: String,
 
         /**
          * Required. A description of the video e.g. 'Gangnam style', or some kind of content description for accessibility e.g. 'video attachment'.
@@ -40,7 +40,7 @@ data class MessageVideoContent(
         @Json(name = "info") val videoInfo: VideoInfo? = null,
 
         /**
-         * Required. Required if the file is unencrypted. The URL (typically MXC URI) to the video clip.
+         * Required if the file is unencrypted. The URL (typically MXC URI) to the video clip.
          */
         @Json(name = "url") override val url: String? = null,
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/read/ReadService.kt

@@ -26,10 +26,16 @@ import im.vector.matrix.android.api.util.Optional
  */
 interface ReadService {
 
+    enum class MarkAsReadParams {
+        READ_RECEIPT,
+        READ_MARKER,
+        BOTH
+    }
+
     /**
      * Force the read marker to be set on the latest event.
      */
-    fun markAllAsRead(callback: MatrixCallback<Unit>)
+    fun markAsRead(params: MarkAsReadParams = MarkAsReadParams.BOTH, callback: MatrixCallback<Unit>)
 
     /**
      * Set the read receipt on the event with provided eventId.

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/send/DraftService.kt

@@ -17,18 +17,20 @@
 package im.vector.matrix.android.api.session.room.send
 
 import androidx.lifecycle.LiveData
+import im.vector.matrix.android.api.MatrixCallback
+import im.vector.matrix.android.api.util.Cancelable
 
 interface DraftService {
 
     /**
      * Save or update a draft to the room
      */
-    fun saveDraft(draft: UserDraft)
+    fun saveDraft(draft: UserDraft, callback: MatrixCallback<Unit>): Cancelable
 
     /**
      * Delete the last draft, basically just after sending the message
      */
-    fun deleteDraft()
+    fun deleteDraft(callback: MatrixCallback<Unit>): Cancelable
 
     /**
      * Return the current drafts if any, as a live data

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/state/StateService.kt

@@ -28,7 +28,7 @@ interface StateService {
      */
     fun updateTopic(topic: String, callback: MatrixCallback<Unit>)
 
-    fun getStateEvent(eventType: String): Event?
+    fun getStateEvent(eventType: String, stateKey: String): Event?
 
-    fun getStateEventLive(eventType: String): LiveData<Optional<Event>>
+    fun getStateEventLive(eventType: String, stateKey: String): LiveData<Optional<Event>>
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/timeline/Timeline.kt

@@ -112,6 +112,11 @@ interface Timeline {
          * Called whenever an error we can't recover from occurred
          */
         fun onTimelineFailure(throwable: Throwable)
+
+        /**
+         * Called when new events come through the sync
+         */
+        fun onNewTimelineEvents(eventIds: List<String>)
     }
 
     /**

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/timeline/TimelineEvent.kt

@@ -36,6 +36,7 @@ import im.vector.matrix.android.internal.crypto.model.event.EncryptedEventConten
 data class TimelineEvent(
         val root: Event,
         val localId: Long,
+        val eventId: String,
         val displayIndex: Int,
         val senderName: String?,
         val isUniqueDisplayName: Boolean,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/DefaultCryptoService.kt

@@ -26,6 +26,7 @@ import com.squareup.moshi.Types
 import com.zhuinden.monarchy.Monarchy
 import dagger.Lazy
 import im.vector.matrix.android.api.MatrixCallback
+import im.vector.matrix.android.api.NoOpMatrixCallback
 import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.api.crypto.MXCryptoConfig
 import im.vector.matrix.android.api.failure.Failure
@@ -71,7 +72,7 @@ import im.vector.matrix.android.internal.crypto.tasks.UploadKeysTask
 import im.vector.matrix.android.internal.crypto.verification.DefaultVerificationService
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
-import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.database.query.whereType
 import im.vector.matrix.android.internal.di.MoshiProvider
 import im.vector.matrix.android.internal.extensions.foldToCallback
 import im.vector.matrix.android.internal.session.SessionScope
@@ -190,7 +191,7 @@ internal class DefaultCryptoService @Inject constructor(
                     this.callback = object : MatrixCallback<Unit> {
                         override fun onSuccess(data: Unit) {
                             // bg refresh of crypto device
-                            downloadKeys(listOf(credentials.userId), true, object : MatrixCallback<MXUsersDevicesMap<CryptoDeviceInfo>> {})
+                            downloadKeys(listOf(credentials.userId), true, NoOpMatrixCallback())
                             callback.onSuccess(data)
                         }
 
@@ -533,7 +534,7 @@ internal class DefaultCryptoService @Inject constructor(
      */
     override fun isRoomEncrypted(roomId: String): Boolean {
         val encryptionEvent = monarchy.fetchCopied { realm ->
-            EventEntity.where(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
+            EventEntity.whereType(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
                     .contains(EventEntityFields.CONTENT, "\"algorithm\":\"$MXCRYPTO_ALGORITHM_MEGOLM\"")
                     .findFirst()
         }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/crosssigning/DefaultCrossSigningService.kt

@@ -36,7 +36,6 @@ import im.vector.matrix.android.internal.crypto.tasks.UploadSignaturesTask
 import im.vector.matrix.android.internal.crypto.tasks.UploadSigningKeysTask
 import im.vector.matrix.android.internal.di.UserId
 import im.vector.matrix.android.internal.session.SessionScope
-import im.vector.matrix.android.internal.task.TaskConstraints
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.configureWith
 import im.vector.matrix.android.internal.util.JsonCanonicalizer
@@ -211,7 +210,6 @@ internal class DefaultCrossSigningService @Inject constructor(
         cryptoStore.storePrivateKeysInfo(masterKeyPrivateKey?.toBase64NoPadding(), uskPrivateKey?.toBase64NoPadding(), sskPrivateKey?.toBase64NoPadding())
 
         uploadSigningKeysTask.configureWith(params) {
-            this.constraints = TaskConstraints(true)
             this.callback = object : MatrixCallback<Unit> {
                 override fun onSuccess(data: Unit) {
                     Timber.i("## CrossSigning - Keys successfully uploaded")
@@ -247,7 +245,6 @@ internal class DefaultCrossSigningService @Inject constructor(
                     resetTrustOnKeyChange()
                     uploadSignaturesTask.configureWith(UploadSignaturesTask.Params(uploadSignatureQueryBuilder.build())) {
                         // this.retryCount = 3
-                        this.constraints = TaskConstraints(true)
                         this.callback = object : MatrixCallback<Unit> {
                             override fun onSuccess(data: Unit) {
                                 Timber.i("## CrossSigning - signatures successfully uploaded")
@@ -396,7 +393,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                         return@forEach
                     } catch (failure: Throwable) {
                         // log
-                        Timber.v(failure)
+                        Timber.w(failure, "Signature not valid?")
                     }
                 }
             }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/keysbackup/KeysBackup.kt

@@ -414,7 +414,7 @@ internal class KeysBackup @Inject constructor(
                             olmDevice.verifySignature(fingerprint, authData.signalableJSONDictionary(), mySignature)
                             isSignatureValid = true
                         } catch (e: OlmException) {
-                            Timber.v(e, "getKeysBackupTrust: Bad signature from device ${device.deviceId}")
+                            Timber.w(e, "getKeysBackupTrust: Bad signature from device ${device.deviceId}")
                         }
                     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/RoomVerificationUpdateTask.kt

@@ -88,7 +88,7 @@ internal class DefaultRoomVerificationUpdateTask @Inject constructor(
                 // done from another device of mine
 
                 if (EventType.MESSAGE == event.type) {
-                    val msgType = event.getClearContent().toModel<MessageContent>()?.type
+                    val msgType = event.getClearContent().toModel<MessageContent>()?.msgType
                     if (MessageType.MSGTYPE_VERIFICATION_REQUEST == msgType) {
                         event.getClearContent().toModel<MessageVerificationRequestContent>()?.let {
                             if (it.fromDevice != deviceId) {
@@ -144,7 +144,7 @@ internal class DefaultRoomVerificationUpdateTask @Inject constructor(
                     params.verificationService.onRoomEvent(event)
                 }
                 EventType.MESSAGE -> {
-                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.type) {
+                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.msgType) {
                         params.verificationService.onRoomRequestReceived(event)
                     }
                 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/UploadSignaturesTask.kt

@@ -37,7 +37,9 @@ internal class DefaultUploadSignaturesTask @Inject constructor(
     override suspend fun execute(params: UploadSignaturesTask.Params) {
         try {
             val response = executeRequest<SignatureUploadResponse>(eventBus) {
-                apiCall = cryptoApi.uploadSignatures(params.signatures)
+                this.isRetryable = true
+                this.maxRetryCount = 10
+                this.apiCall = cryptoApi.uploadSignatures(params.signatures)
             }
             if (response.failures?.isNotEmpty() == true) {
                 throw Throwable(response.failures.toString())

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/verification/DefaultVerificationService.kt

@@ -165,7 +165,7 @@ internal class DefaultVerificationService @Inject constructor(
                     onRoomDoneReceived(event)
                 }
                 EventType.MESSAGE                 -> {
-                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.type) {
+                    if (MessageType.MSGTYPE_VERIFICATION_REQUEST == event.getClearContent().toModel<MessageContent>()?.msgType) {
                         onRoomRequestReceived(event)
                     }
                 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/verification/VerificationMessageLiveObserver.kt

@@ -24,7 +24,7 @@ import im.vector.matrix.android.internal.crypto.tasks.RoomVerificationUpdateTask
 import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.di.SessionDatabase
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.configureWith
@@ -42,7 +42,7 @@ internal class VerificationMessageLiveObserver @Inject constructor(
 ) : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query {
-        EventEntity.types(it, listOf(
+        EventEntity.whereTypes(it, listOf(
                 EventType.KEY_VERIFICATION_START,
                 EventType.KEY_VERIFICATION_ACCEPT,
                 EventType.KEY_VERIFICATION_KEY,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/verification/VerificationTransportRoomMessage.kt

@@ -331,7 +331,7 @@ internal class VerificationTransportRoomMessage(
                 content = content,
                 unsignedData = UnsignedData(age = null, transactionId = localID)
         ).also {
-            localEchoEventFactory.saveLocalEcho(monarchy, it)
+            localEchoEventFactory.createLocalEcho(it)
         }
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt

@@ -16,19 +16,27 @@
 
 package im.vector.matrix.android.internal.database.helper
 
-import im.vector.matrix.android.api.session.events.model.Event
-import im.vector.matrix.android.api.session.events.model.EventType
-import im.vector.matrix.android.api.session.room.send.SendState
-import im.vector.matrix.android.internal.database.mapper.asDomain
-import im.vector.matrix.android.internal.database.mapper.toEntity
-import im.vector.matrix.android.internal.database.model.*
+import im.vector.matrix.android.api.session.room.model.RoomMemberContent
+import im.vector.matrix.android.internal.database.model.ChunkEntity
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntityFields
+import im.vector.matrix.android.internal.database.model.EventAnnotationsSummaryEntity
+import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.EventEntityFields
+import im.vector.matrix.android.internal.database.model.ReadReceiptEntity
+import im.vector.matrix.android.internal.database.model.ReadReceiptsSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields
 import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.getOrCreate
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.extensions.assertIsManaged
 import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
+import io.realm.Realm
 import io.realm.Sort
 import io.realm.kotlin.createObject
+import timber.log.Timber
 
 internal fun ChunkEntity.deleteOnCascade() {
     assertIsManaged()
@@ -36,116 +44,154 @@ internal fun ChunkEntity.deleteOnCascade() {
     this.deleteFromRealm()
 }
 
-internal fun ChunkEntity.merge(roomId: String,
-                               chunkToMerge: ChunkEntity,
-                               direction: PaginationDirection): List<TimelineEventEntity> {
+internal fun ChunkEntity.merge(roomId: String, chunkToMerge: ChunkEntity, direction: PaginationDirection) {
     assertIsManaged()
-    val isChunkToMergeUnlinked = chunkToMerge.isUnlinked
-    val isCurrentChunkUnlinked = isUnlinked
-
-    if (isCurrentChunkUnlinked && !isChunkToMergeUnlinked) {
-        this.timelineEvents.forEach { it.root?.isUnlinked = false }
-    }
+    val localRealm = this.realm
     val eventsToMerge: List<TimelineEventEntity>
     if (direction == PaginationDirection.FORWARDS) {
         this.nextToken = chunkToMerge.nextToken
         this.isLastForward = chunkToMerge.isLastForward
-        eventsToMerge = chunkToMerge.timelineEvents.sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.ASCENDING)
+        eventsToMerge = chunkToMerge.timelineEvents.sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.ASCENDING)
     } else {
         this.prevToken = chunkToMerge.prevToken
         this.isLastBackward = chunkToMerge.isLastBackward
-        eventsToMerge = chunkToMerge.timelineEvents.sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.DESCENDING)
+        eventsToMerge = chunkToMerge.timelineEvents.sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING)
+    }
+    chunkToMerge.stateEvents.forEach { stateEvent ->
+        addStateEvent(roomId, stateEvent, direction)
     }
     return eventsToMerge
-            .mapNotNull {
-                val event = it.root?.asDomain() ?: return@mapNotNull null
-                add(roomId, event, direction)
+            .forEach {
+                addTimelineEventFromMerge(localRealm, it, direction)
             }
 }
 
-internal fun ChunkEntity.add(roomId: String,
-                             event: Event,
-                             direction: PaginationDirection,
-                             stateIndexOffset: Int = 0
-): TimelineEventEntity? {
-    assertIsManaged()
-    if (event.eventId != null && timelineEvents.find(event.eventId) != null) {
-        return null
-    }
-    var currentDisplayIndex = lastDisplayIndex(direction, 0)
-    if (direction == PaginationDirection.FORWARDS) {
-        currentDisplayIndex += 1
-        forwardsDisplayIndex = currentDisplayIndex
+internal fun ChunkEntity.addStateEvent(roomId: String, stateEvent: EventEntity, direction: PaginationDirection) {
+    if (direction == PaginationDirection.BACKWARDS) {
+        Timber.v("We don't keep chunk state events when paginating backward")
     } else {
-        currentDisplayIndex -= 1
-        backwardsDisplayIndex = currentDisplayIndex
-    }
-    var currentStateIndex = lastStateIndex(direction, defaultValue = stateIndexOffset)
-    if (direction == PaginationDirection.FORWARDS && EventType.isStateEvent(event.type)) {
-        currentStateIndex += 1
-        forwardsStateIndex = currentStateIndex
-    } else if (direction == PaginationDirection.BACKWARDS && timelineEvents.isNotEmpty()) {
-        val lastEventType = timelineEvents.last()?.root?.type ?: ""
-        if (EventType.isStateEvent(lastEventType)) {
-            currentStateIndex -= 1
-            backwardsStateIndex = currentStateIndex
+        val stateKey = stateEvent.stateKey ?: return
+        val type = stateEvent.type
+        val pastStateEvent = stateEvents.where()
+                .equalTo(EventEntityFields.ROOM_ID, roomId)
+                .equalTo(EventEntityFields.STATE_KEY, stateKey)
+                .equalTo(CurrentStateEventEntityFields.TYPE, type)
+                .findFirst()
+
+        if (pastStateEvent != null) {
+            stateEvents.remove(pastStateEvent)
         }
+        stateEvents.add(stateEvent)
     }
+}
 
-    val isChunkUnlinked = isUnlinked
+internal fun ChunkEntity.addTimelineEvent(roomId: String,
+                                          eventEntity: EventEntity,
+                                          direction: PaginationDirection,
+                                          roomMemberContentsByUser: Map<String, RoomMemberContent?>) {
+    val eventId = eventEntity.eventId
+    if (timelineEvents.find(eventId) != null) {
+        return
+    }
+    val displayIndex = nextDisplayIndex(direction)
     val localId = TimelineEventEntity.nextId(realm)
-    val eventId = event.eventId ?: ""
-    val senderId = event.senderId ?: ""
-
-    val readReceiptsSummaryEntity = ReadReceiptsSummaryEntity.where(realm, eventId).findFirst()
-            ?: realm.createObject<ReadReceiptsSummaryEntity>(eventId).apply {
-                this.roomId = roomId
-            }
+    val senderId = eventEntity.sender ?: ""
 
     // Update RR for the sender of a new message with a dummy one
+    val readReceiptsSummaryEntity = handleReadReceipts(realm, roomId, eventEntity, senderId)
+    val timelineEventEntity = realm.createObject<TimelineEventEntity>().apply {
+        this.localId = localId
+        this.root = eventEntity
+        this.eventId = eventId
+        this.roomId = roomId
+        this.annotations = EventAnnotationsSummaryEntity.where(realm, eventId).findFirst()
+        this.readReceipts = readReceiptsSummaryEntity
+        this.displayIndex = displayIndex
+        val roomMemberContent = roomMemberContentsByUser[senderId]
+        this.senderAvatar = roomMemberContent?.avatarUrl
+        this.senderName = roomMemberContent?.displayName
+        isUniqueDisplayName = if (roomMemberContent?.displayName != null) {
+            computeIsUnique(realm, roomId, isLastForward, roomMemberContent, roomMemberContentsByUser)
+        } else {
+            true
+        }
+    }
+    timelineEvents.add(timelineEventEntity)
+}
 
-    if (event.originServerTs != null) {
-        val timestampOfEvent = event.originServerTs.toDouble()
+private fun computeIsUnique(
+        realm: Realm,
+        roomId: String,
+        isLastForward: Boolean,
+        myRoomMemberContent: RoomMemberContent,
+        roomMemberContentsByUser: Map<String, RoomMemberContent?>
+): Boolean {
+    val isHistoricalUnique = roomMemberContentsByUser.values.find {
+        it != myRoomMemberContent && it?.displayName == myRoomMemberContent.displayName
+    } == null
+    return if (isLastForward) {
+        val isLiveUnique = RoomMemberSummaryEntity
+                .where(realm, roomId)
+                .equalTo(RoomMemberSummaryEntityFields.DISPLAY_NAME, myRoomMemberContent.displayName)
+                .findAll().none {
+                    !roomMemberContentsByUser.containsKey(it.userId)
+                }
+        isHistoricalUnique && isLiveUnique
+    } else {
+        isHistoricalUnique
+    }
+}
+
+private fun ChunkEntity.addTimelineEventFromMerge(realm: Realm, timelineEventEntity: TimelineEventEntity, direction: PaginationDirection) {
+    val eventId = timelineEventEntity.eventId
+    if (timelineEvents.find(eventId) != null) {
+        return
+    }
+    val displayIndex = nextDisplayIndex(direction)
+    val localId = TimelineEventEntity.nextId(realm)
+    val copied = realm.createObject<TimelineEventEntity>().apply {
+        this.localId = localId
+        this.root = timelineEventEntity.root
+        this.eventId = timelineEventEntity.eventId
+        this.roomId = timelineEventEntity.roomId
+        this.annotations = timelineEventEntity.annotations
+        this.readReceipts = timelineEventEntity.readReceipts
+        this.displayIndex = displayIndex
+        this.senderAvatar = timelineEventEntity.senderAvatar
+        this.senderName = timelineEventEntity.senderName
+        this.isUniqueDisplayName = timelineEventEntity.isUniqueDisplayName
+    }
+    timelineEvents.add(copied)
+}
+
+private fun handleReadReceipts(realm: Realm, roomId: String, eventEntity: EventEntity, senderId: String): ReadReceiptsSummaryEntity {
+    val readReceiptsSummaryEntity = ReadReceiptsSummaryEntity.where(realm, eventEntity.eventId).findFirst()
+            ?: realm.createObject<ReadReceiptsSummaryEntity>(eventEntity.eventId).apply {
+                this.roomId = roomId
+            }
+    val originServerTs = eventEntity.originServerTs
+    if (originServerTs != null) {
+        val timestampOfEvent = originServerTs.toDouble()
         val readReceiptOfSender = ReadReceiptEntity.getOrCreate(realm, roomId = roomId, userId = senderId)
         // If the synced RR is older, update
         if (timestampOfEvent > readReceiptOfSender.originServerTs) {
             val previousReceiptsSummary = ReadReceiptsSummaryEntity.where(realm, eventId = readReceiptOfSender.eventId).findFirst()
-            readReceiptOfSender.eventId = eventId
+            readReceiptOfSender.eventId = eventEntity.eventId
             readReceiptOfSender.originServerTs = timestampOfEvent
             previousReceiptsSummary?.readReceipts?.remove(readReceiptOfSender)
             readReceiptsSummaryEntity.readReceipts.add(readReceiptOfSender)
         }
     }
-
-    val rootEvent = event.toEntity(roomId).apply {
-        this.stateIndex = currentStateIndex
-        this.displayIndex = currentDisplayIndex
-        this.sendState = SendState.SYNCED
-        this.isUnlinked = isChunkUnlinked
-    }
-    val eventEntity = realm.createObject<TimelineEventEntity>().also {
-        it.localId = localId
-        it.root = realm.copyToRealm(rootEvent)
-        it.eventId = eventId
-        it.roomId = roomId
-        it.annotations = EventAnnotationsSummaryEntity.where(realm, eventId).findFirst()
-        it.readReceipts = readReceiptsSummaryEntity
-    }
-    val position = if (direction == PaginationDirection.FORWARDS) 0 else this.timelineEvents.size
-    timelineEvents.add(position, eventEntity)
-    return eventEntity
+    return readReceiptsSummaryEntity
 }
 
-internal fun ChunkEntity.lastDisplayIndex(direction: PaginationDirection, defaultValue: Int = 0): Int {
+internal fun ChunkEntity.nextDisplayIndex(direction: PaginationDirection): Int {
     return when (direction) {
-        PaginationDirection.FORWARDS  -> forwardsDisplayIndex
-        PaginationDirection.BACKWARDS -> backwardsDisplayIndex
-    } ?: defaultValue
-}
-
-internal fun ChunkEntity.lastStateIndex(direction: PaginationDirection, defaultValue: Int = 0): Int {
-    return when (direction) {
-        PaginationDirection.FORWARDS  -> forwardsStateIndex
-        PaginationDirection.BACKWARDS -> backwardsStateIndex
-    } ?: defaultValue
+        PaginationDirection.FORWARDS  -> {
+            (timelineEvents.where().max(TimelineEventEntityFields.DISPLAY_INDEX)?.toInt() ?: 0) + 1
+        }
+        PaginationDirection.BACKWARDS -> {
+            (timelineEvents.where().min(TimelineEventEntityFields.DISPLAY_INDEX)?.toInt() ?: 0) - 1
+        }
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/RoomEntityHelper.kt

@@ -16,15 +16,8 @@
 
 package im.vector.matrix.android.internal.database.helper
 
-import im.vector.matrix.android.api.session.events.model.Event
-import im.vector.matrix.android.api.session.room.send.SendState
-import im.vector.matrix.android.internal.database.mapper.toEntity
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
-import im.vector.matrix.android.internal.database.model.TimelineEventEntity
-import im.vector.matrix.android.internal.database.query.fastContains
-import im.vector.matrix.android.internal.extensions.assertIsManaged
-import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
 
 internal fun RoomEntity.deleteOnCascade(chunkEntity: ChunkEntity) {
     chunks.remove(chunkEntity)
@@ -36,39 +29,3 @@ internal fun RoomEntity.addOrUpdate(chunkEntity: ChunkEntity) {
         chunks.add(chunkEntity)
     }
 }
-
-internal fun RoomEntity.addStateEvent(stateEvent: Event,
-                                      stateIndex: Int = Int.MIN_VALUE,
-                                      filterDuplicates: Boolean = false,
-                                      isUnlinked: Boolean = false) {
-    assertIsManaged()
-    if (stateEvent.eventId == null || (filterDuplicates && fastContains(stateEvent.eventId))) {
-        return
-    } else {
-        val entity = stateEvent.toEntity(roomId).apply {
-            this.stateIndex = stateIndex
-            this.isUnlinked = isUnlinked
-            this.sendState = SendState.SYNCED
-        }
-        untimelinedStateEvents.add(entity)
-    }
-}
-internal fun RoomEntity.addSendingEvent(event: Event) {
-    assertIsManaged()
-    val senderId = event.senderId ?: return
-    val eventEntity = event.toEntity(roomId).apply {
-        this.sendState = SendState.UNSENT
-    }
-    val roomMembers = RoomMemberHelper(realm, roomId)
-    val myUser = roomMembers.getLastRoomMember(senderId)
-    val localId = TimelineEventEntity.nextId(realm)
-    val timelineEventEntity = TimelineEventEntity(localId).also {
-        it.root = eventEntity
-        it.eventId = event.eventId ?: ""
-        it.roomId = roomId
-        it.senderName = myUser?.displayName
-        it.senderAvatar = myUser?.avatarUrl
-        it.isUniqueDisplayName = roomMembers.isUniqueDisplayName(myUser?.displayName)
-    }
-    sendingTimelineEvents.add(0, timelineEventEntity)
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/TimelineEventSenderVisitor.kt

@@ -1,148 +0,0 @@
-/*
- * Copyright 2019 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package im.vector.matrix.android.internal.database.helper
-
-import im.vector.matrix.android.api.session.events.model.EventType
-import im.vector.matrix.android.api.session.events.model.toModel
-import im.vector.matrix.android.api.session.room.model.RoomMemberContent
-import im.vector.matrix.android.internal.database.mapper.ContentMapper
-import im.vector.matrix.android.internal.database.model.*
-import im.vector.matrix.android.internal.database.query.next
-import im.vector.matrix.android.internal.database.query.prev
-import im.vector.matrix.android.internal.database.query.where
-import im.vector.matrix.android.internal.extensions.assertIsManaged
-import im.vector.matrix.android.internal.session.SessionScope
-import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
-import io.realm.RealmList
-import io.realm.RealmQuery
-import javax.inject.Inject
-
-/**
- * This is an internal cache to avoid querying all the time the room member events
- */
-@SessionScope
-internal class TimelineEventSenderVisitor @Inject constructor() {
-
-    internal data class Key(
-            val roomId: String,
-            val stateIndex: Int,
-            val senderId: String
-    )
-
-    internal class Value(
-            var senderAvatar: String? = null,
-            var senderName: String? = null,
-            var isUniqueDisplayName: Boolean = false,
-            var senderMembershipEventId: String? = null
-    )
-
-    private val values = HashMap<Key, Value>()
-
-    fun clear() {
-        values.clear()
-    }
-
-    fun clear(roomId: String, senderId: String) {
-        val keysToRemove = values.keys.filter { it.senderId == senderId && it.roomId == roomId }
-        keysToRemove.forEach {
-            values.remove(it)
-        }
-    }
-
-    fun visit(timelineEventEntities: List<TimelineEventEntity>) = timelineEventEntities.forEach { visit(it) }
-
-    fun visit(timelineEventEntity: TimelineEventEntity) {
-        if (!timelineEventEntity.isValid) {
-            return
-        }
-        val key = Key(
-                roomId = timelineEventEntity.roomId,
-                stateIndex = timelineEventEntity.root?.stateIndex ?: 0,
-                senderId = timelineEventEntity.root?.sender ?: ""
-        )
-        val result = values.getOrPut(key) {
-            timelineEventEntity.computeValue()
-        }
-        timelineEventEntity.apply {
-            this.isUniqueDisplayName = result.isUniqueDisplayName
-            this.senderAvatar = result.senderAvatar
-            this.senderName = result.senderName
-            this.senderMembershipEventId = result.senderMembershipEventId
-        }
-    }
-
-    private fun RealmList<TimelineEventEntity>.buildQuery(sender: String, isUnlinked: Boolean): RealmQuery<TimelineEventEntity> {
-        return where()
-                .equalTo(TimelineEventEntityFields.ROOT.STATE_KEY, sender)
-                .equalTo(TimelineEventEntityFields.ROOT.TYPE, EventType.STATE_ROOM_MEMBER)
-                .equalTo(TimelineEventEntityFields.ROOT.IS_UNLINKED, isUnlinked)
-    }
-
-    private fun TimelineEventEntity.computeValue(): Value {
-        assertIsManaged()
-        val result = Value()
-        val roomEntity = RoomEntity.where(realm, roomId = roomId).findFirst() ?: return result
-        val stateIndex = root?.stateIndex ?: return result
-        val senderId = root?.sender ?: return result
-        val chunkEntity = chunk?.firstOrNull() ?: return result
-        val isUnlinked = chunkEntity.isUnlinked
-        var senderMembershipEvent: EventEntity?
-        var senderRoomMemberContent: String?
-        var senderRoomMemberPrevContent: String?
-
-        if (stateIndex <= 0) {
-            senderMembershipEvent = chunkEntity.timelineEvents.buildQuery(senderId, isUnlinked).next(from = stateIndex)?.root
-            senderRoomMemberContent = senderMembershipEvent?.prevContent
-            senderRoomMemberPrevContent = senderMembershipEvent?.content
-        } else {
-            senderMembershipEvent = chunkEntity.timelineEvents.buildQuery(senderId, isUnlinked).prev(since = stateIndex)?.root
-            senderRoomMemberContent = senderMembershipEvent?.content
-            senderRoomMemberPrevContent = senderMembershipEvent?.prevContent
-        }
-
-        // We fallback to untimelinedStateEvents if we can't find membership events in timeline
-        if (senderMembershipEvent == null) {
-            senderMembershipEvent = roomEntity.untimelinedStateEvents
-                    .where()
-                    .equalTo(EventEntityFields.STATE_KEY, senderId)
-                    .equalTo(EventEntityFields.TYPE, EventType.STATE_ROOM_MEMBER)
-                    .prev(since = stateIndex)
-            senderRoomMemberContent = senderMembershipEvent?.content
-            senderRoomMemberPrevContent = senderMembershipEvent?.prevContent
-        }
-
-        ContentMapper.map(senderRoomMemberContent).toModel<RoomMemberContent>()?.also {
-            result.senderAvatar = it.avatarUrl
-            result.senderName = it.displayName
-            result.isUniqueDisplayName = RoomMemberHelper(realm, roomId).isUniqueDisplayName(it.displayName)
-        }
-        // We try to fallback on prev content if we got a room member state events with null fields
-        if (root?.type == EventType.STATE_ROOM_MEMBER) {
-            ContentMapper.map(senderRoomMemberPrevContent).toModel<RoomMemberContent>()?.also {
-                if (result.senderAvatar == null && it.avatarUrl != null) {
-                    result.senderAvatar = it.avatarUrl
-                }
-                if (result.senderName == null && it.displayName != null) {
-                    result.senderName = it.displayName
-                    result.isUniqueDisplayName = RoomMemberHelper(realm, roomId).isUniqueDisplayName(it.displayName)
-                }
-            }
-        }
-        result.senderMembershipEventId = senderMembershipEvent?.eventId
-        return result
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/EventMapper.kt

@@ -20,6 +20,7 @@ import com.squareup.moshi.JsonDataException
 import im.vector.matrix.android.api.session.crypto.MXCryptoError
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.events.model.UnsignedData
+import im.vector.matrix.android.api.session.room.send.SendState
 import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.di.MoshiProvider
@@ -43,7 +44,6 @@ internal object EventMapper {
         eventEntity.redacts = event.redacts
         eventEntity.age = event.unsignedData?.age ?: event.originServerTs
         eventEntity.unsignedData = uds
-        eventEntity.ageLocalTs = event.ageLocalTs
         return eventEntity
     }
 
@@ -92,6 +92,9 @@ internal fun EventEntity.asDomain(): Event {
     return EventMapper.map(this)
 }
 
-internal fun Event.toEntity(roomId: String): EventEntity {
-    return EventMapper.map(this, roomId)
+internal fun Event.toEntity(roomId: String, sendState: SendState, ageLocalTs: Long? = null): EventEntity {
+    return EventMapper.map(this, roomId).apply {
+        this.sendState = sendState
+        this.ageLocalTs = ageLocalTs
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/TimelineEventMapper.kt

@@ -37,15 +37,19 @@ internal class TimelineEventMapper @Inject constructor(private val readReceiptsS
         return TimelineEvent(
                 root = timelineEventEntity.root?.asDomain()
                         ?: Event("", timelineEventEntity.eventId),
+                eventId = timelineEventEntity.eventId,
                 annotations = timelineEventEntity.annotations?.asDomain(),
                 localId = timelineEventEntity.localId,
-                displayIndex = timelineEventEntity.root?.displayIndex ?: 0,
+                displayIndex = timelineEventEntity.displayIndex,
                 senderName = timelineEventEntity.senderName,
                 isUniqueDisplayName = timelineEventEntity.isUniqueDisplayName,
                 senderAvatar = timelineEventEntity.senderAvatar,
-                readReceipts = readReceipts?.sortedByDescending {
-                    it.originServerTs
-                } ?: emptyList()
+                readReceipts = readReceipts
+                        ?.distinctBy {
+                            it.user
+                        }?.sortedByDescending {
+                            it.originServerTs
+                        } ?: emptyList()
         )
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt

@@ -24,14 +24,10 @@ import io.realm.annotations.LinkingObjects
 
 internal open class ChunkEntity(@Index var prevToken: String? = null,
                                 @Index var nextToken: String? = null,
+                                var stateEvents: RealmList<EventEntity> = RealmList(),
                                 var timelineEvents: RealmList<TimelineEventEntity> = RealmList(),
                                 @Index var isLastForward: Boolean = false,
-                                @Index var isLastBackward: Boolean = false,
-                                var backwardsDisplayIndex: Int? = null,
-                                var forwardsDisplayIndex: Int? = null,
-                                var backwardsStateIndex: Int? = null,
-                                var forwardsStateIndex: Int? = null,
-                                var isUnlinked: Boolean = false
+                                @Index var isLastBackward: Boolean = false
 ) : RealmObject() {
 
     fun identifier() = "${prevToken}_$nextToken"

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/CurrentStateEventEntity.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package im.vector.matrix.android.internal.database.model
+
+import io.realm.RealmObject
+import io.realm.annotations.Index
+
+internal open class CurrentStateEventEntity(var eventId: String = "",
+                                            var root: EventEntity? = null,
+                                            @Index var roomId: String = "",
+                                            @Index var type: String = "",
+                                            @Index var stateKey: String = ""
+) : RealmObject() {
+    companion object
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/EventEntity.kt

@@ -21,11 +21,10 @@ import im.vector.matrix.android.internal.crypto.MXEventDecryptionResult
 import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
 import im.vector.matrix.android.internal.di.MoshiProvider
 import io.realm.RealmObject
-import io.realm.RealmResults
 import io.realm.annotations.Index
-import io.realm.annotations.LinkingObjects
+import io.realm.annotations.PrimaryKey
 
-internal open class EventEntity(@Index var eventId: String = "",
+internal open class EventEntity(@PrimaryKey var eventId: String = "",
                                 @Index var roomId: String = "",
                                 @Index var type: String = "",
                                 var content: String? = null,
@@ -36,20 +35,11 @@ internal open class EventEntity(@Index var eventId: String = "",
                                 var age: Long? = 0,
                                 var unsignedData: String? = null,
                                 var redacts: String? = null,
-                                @Index var stateIndex: Int = 0,
-                                @Index var displayIndex: Int = 0,
-                                @Index var isUnlinked: Boolean = false,
                                 var decryptionResultJson: String? = null,
                                 var decryptionErrorCode: String? = null,
                                 var ageLocalTs: Long? = null
 ) : RealmObject() {
 
-    enum class LinkFilterMode {
-        LINKED_ONLY,
-        UNLINKED_ONLY,
-        BOTH
-    }
-
     private var sendStateStr: String = SendState.UNKNOWN.name
 
     var sendState: SendState
@@ -62,12 +52,6 @@ internal open class EventEntity(@Index var eventId: String = "",
 
     companion object
 
-    @LinkingObjects("untimelinedStateEvents")
-    val room: RealmResults<RoomEntity>? = null
-
-    @LinkingObjects("root")
-    val timelineEventEntity: RealmResults<TimelineEventEntity>? = null
-
     fun setDecryptionResult(result: MXEventDecryptionResult) {
         val decryptionResult = OlmDecryptionResult(
                 payload = result.clearEvent,
@@ -78,6 +62,5 @@ internal open class EventEntity(@Index var eventId: String = "",
         val adapter = MoshiProvider.providesMoshi().adapter<OlmDecryptionResult>(OlmDecryptionResult::class.java)
         decryptionResultJson = adapter.toJson(decryptionResult)
         decryptionErrorCode = null
-        timelineEventEntity?.firstOrNull()?.root = this
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/RoomEntity.kt

@@ -23,7 +23,6 @@ import io.realm.annotations.PrimaryKey
 
 internal open class RoomEntity(@PrimaryKey var roomId: String = "",
                                var chunks: RealmList<ChunkEntity> = RealmList(),
-                               var untimelinedStateEvents: RealmList<EventEntity> = RealmList(),
                                var sendingTimelineEvents: RealmList<TimelineEventEntity> = RealmList(),
                                var areAllMembersLoaded: Boolean = false
 ) : RealmObject() {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/SessionRealmModule.kt

@@ -22,35 +22,36 @@ import io.realm.annotations.RealmModule
  * Realm module for Session
  */
 @RealmModule(library = true,
-             classes = [
-                 ChunkEntity::class,
-                 EventEntity::class,
-                 TimelineEventEntity::class,
-                 FilterEntity::class,
-                 GroupEntity::class,
-                 GroupSummaryEntity::class,
-                 ReadReceiptEntity::class,
-                 RoomEntity::class,
-                 RoomSummaryEntity::class,
-                 RoomTagEntity::class,
-                 SyncEntity::class,
-                 UserEntity::class,
-                 IgnoredUserEntity::class,
-                 BreadcrumbsEntity::class,
-                 EventAnnotationsSummaryEntity::class,
-                 ReferencesAggregatedSummaryEntity::class,
-                 ReactionAggregatedSummaryEntity::class,
-                 EditAggregatedSummaryEntity::class,
-                 PushRulesEntity::class,
-                 PushRuleEntity::class,
-                 PushConditionEntity::class,
-                 PusherEntity::class,
-                 PusherDataEntity::class,
-                 ReadReceiptsSummaryEntity::class,
-                 ReadMarkerEntity::class,
-                 UserDraftsEntity::class,
-                 DraftEntity::class,
-                 HomeServerCapabilitiesEntity::class,
-                 RoomMemberSummaryEntity::class
-             ])
+        classes = [
+            ChunkEntity::class,
+            EventEntity::class,
+            TimelineEventEntity::class,
+            FilterEntity::class,
+            GroupEntity::class,
+            GroupSummaryEntity::class,
+            ReadReceiptEntity::class,
+            RoomEntity::class,
+            RoomSummaryEntity::class,
+            RoomTagEntity::class,
+            SyncEntity::class,
+            UserEntity::class,
+            IgnoredUserEntity::class,
+            BreadcrumbsEntity::class,
+            EventAnnotationsSummaryEntity::class,
+            ReactionAggregatedSummaryEntity::class,
+            EditAggregatedSummaryEntity::class,
+            ReferencesAggregatedSummaryEntity::class,
+            PushRulesEntity::class,
+            PushRuleEntity::class,
+            PushConditionEntity::class,
+            PusherEntity::class,
+            PusherDataEntity::class,
+            ReadReceiptsSummaryEntity::class,
+            ReadMarkerEntity::class,
+            UserDraftsEntity::class,
+            DraftEntity::class,
+            HomeServerCapabilitiesEntity::class,
+            RoomMemberSummaryEntity::class,
+            CurrentStateEventEntity::class
+        ])
 internal class SessionRealmModule

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/TimelineEventEntity.kt

@@ -24,6 +24,7 @@ import io.realm.annotations.LinkingObjects
 internal open class TimelineEventEntity(var localId: Long = 0,
                                         @Index var eventId: String = "",
                                         @Index var roomId: String = "",
+                                        @Index var displayIndex: Int = 0,
                                         var root: EventEntity? = null,
                                         var annotations: EventAnnotationsSummaryEntity? = null,
                                         var senderName: String? = null,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/ChunkEntityQueries.kt

@@ -60,12 +60,10 @@ internal fun ChunkEntity.Companion.findIncludingEvent(realm: Realm, eventId: Str
 internal fun ChunkEntity.Companion.create(
         realm: Realm,
         prevToken: String?,
-        nextToken: String?,
-        isUnlinked: Boolean
+        nextToken: String?
 ): ChunkEntity {
     return realm.createObject<ChunkEntity>().apply {
         this.prevToken = prevToken
         this.nextToken = nextToken
-        this.isUnlinked = isUnlinked
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/CurrentStateEventEntityQueries.kt

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package im.vector.matrix.android.internal.database.query
+
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntityFields
+import io.realm.Realm
+import io.realm.RealmQuery
+import io.realm.kotlin.createObject
+
+internal fun CurrentStateEventEntity.Companion.where(realm: Realm, roomId: String, type: String): RealmQuery<CurrentStateEventEntity> {
+    return realm.where(CurrentStateEventEntity::class.java)
+            .equalTo(CurrentStateEventEntityFields.ROOM_ID, roomId)
+            .equalTo(CurrentStateEventEntityFields.TYPE, type)
+}
+
+internal fun CurrentStateEventEntity.Companion.whereStateKey(realm: Realm, roomId: String, type: String, stateKey: String)
+        : RealmQuery<CurrentStateEventEntity> {
+    return where(realm = realm, roomId = roomId, type = type)
+            .equalTo(CurrentStateEventEntityFields.STATE_KEY, stateKey)
+}
+
+internal fun CurrentStateEventEntity.Companion.getOrNull(realm: Realm, roomId: String, stateKey: String, type: String): CurrentStateEventEntity? {
+    return whereStateKey(realm = realm, roomId = roomId, type = type, stateKey = stateKey).findFirst()
+}
+
+internal fun CurrentStateEventEntity.Companion.getOrCreate(realm: Realm, roomId: String, stateKey: String, type: String): CurrentStateEventEntity {
+    return getOrNull(realm = realm, roomId = roomId, stateKey = stateKey, type = type) ?: create(realm, roomId, stateKey, type)
+}
+
+private fun create(realm: Realm, roomId: String, stateKey: String, type: String): CurrentStateEventEntity {
+    return realm.createObject<CurrentStateEventEntity>().apply {
+        this.type = type
+        this.roomId = roomId
+        this.stateKey = stateKey
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/EventEntityQueries.kt

@@ -17,12 +17,10 @@
 package im.vector.matrix.android.internal.database.query
 
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.model.EventEntity.LinkFilterMode.*
 import im.vector.matrix.android.internal.database.model.EventEntityFields
 import io.realm.Realm
 import io.realm.RealmList
 import io.realm.RealmQuery
-import io.realm.Sort
 import io.realm.kotlin.where
 
 internal fun EventEntity.Companion.where(realm: Realm, eventId: String): RealmQuery<EventEntity> {
@@ -35,61 +33,28 @@ internal fun EventEntity.Companion.where(realm: Realm, eventIds: List<String>):
             .`in`(EventEntityFields.EVENT_ID, eventIds.toTypedArray())
 }
 
-internal fun EventEntity.Companion.where(realm: Realm,
-                                         roomId: String? = null,
-                                         type: String? = null,
-                                         linkFilterMode: EventEntity.LinkFilterMode = LINKED_ONLY): RealmQuery<EventEntity> {
+internal fun EventEntity.Companion.whereType(realm: Realm,
+                                             type: String,
+                                             roomId: String? = null
+): RealmQuery<EventEntity> {
     val query = realm.where<EventEntity>()
     if (roomId != null) {
         query.equalTo(EventEntityFields.ROOM_ID, roomId)
     }
-    if (type != null) {
-        query.equalTo(EventEntityFields.TYPE, type)
-    }
-    return when (linkFilterMode) {
-        LINKED_ONLY   -> query.equalTo(EventEntityFields.IS_UNLINKED, false)
-        UNLINKED_ONLY -> query.equalTo(EventEntityFields.IS_UNLINKED, true)
-        BOTH          -> query
-    }
+    return query.equalTo(EventEntityFields.TYPE, type)
 }
 
-internal fun EventEntity.Companion.types(realm: Realm,
-                                         typeList: List<String> = emptyList()): RealmQuery<EventEntity> {
+internal fun EventEntity.Companion.whereTypes(realm: Realm,
+                                              typeList: List<String> = emptyList(),
+                                              roomId: String? = null): RealmQuery<EventEntity> {
     val query = realm.where<EventEntity>()
     query.`in`(EventEntityFields.TYPE, typeList.toTypedArray())
+    if (roomId != null) {
+        query.equalTo(EventEntityFields.ROOM_ID, roomId)
+    }
     return query
 }
 
-internal fun RealmQuery<EventEntity>.descending(since: Int? = null, strict: Boolean = false): RealmQuery<EventEntity> {
-    if (since != null) {
-        if (strict) {
-            this.lessThan(EventEntityFields.STATE_INDEX, since)
-        } else {
-            this.lessThanOrEqualTo(EventEntityFields.STATE_INDEX, since)
-        }
-    }
-    return this.sort(EventEntityFields.STATE_INDEX, Sort.DESCENDING)
-}
-
-internal fun RealmQuery<EventEntity>.ascending(from: Int? = null, strict: Boolean = true): RealmQuery<EventEntity> {
-    if (from != null) {
-        if (strict) {
-            this.greaterThan(EventEntityFields.STATE_INDEX, from)
-        } else {
-            this.greaterThanOrEqualTo(EventEntityFields.STATE_INDEX, from)
-        }
-    }
-    return this.sort(EventEntityFields.STATE_INDEX, Sort.ASCENDING)
-}
-
-internal fun RealmQuery<EventEntity>.next(from: Int? = null, strict: Boolean = true): EventEntity? {
-    return this.ascending(from, strict).findFirst()
-}
-
-internal fun RealmQuery<EventEntity>.prev(since: Int? = null, strict: Boolean = false): EventEntity? {
-    return descending(since, strict).findFirst()
-}
-
 internal fun RealmList<EventEntity>.find(eventId: String): EventEntity? {
     return this.where()
             .equalTo(EventEntityFields.EVENT_ID, eventId)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/ReadQueries.kt

@@ -20,6 +20,7 @@ import im.vector.matrix.android.api.session.events.model.LocalEcho
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.ReadMarkerEntity
 import im.vector.matrix.android.internal.database.model.ReadReceiptEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
 import io.realm.Realm
 
 internal fun isEventRead(monarchy: Monarchy,
@@ -36,16 +37,15 @@ internal fun isEventRead(monarchy: Monarchy,
 
     monarchy.doWithRealm { realm ->
         val liveChunk = ChunkEntity.findLastLiveChunkFromRoom(realm, roomId) ?: return@doWithRealm
-        val eventToCheck = liveChunk.timelineEvents.find(eventId)?.root
-
-        isEventRead = if (eventToCheck?.sender == userId) {
+        val eventToCheck = liveChunk.timelineEvents.find(eventId)
+        isEventRead = if (eventToCheck == null || eventToCheck.root?.sender == userId) {
             true
         } else {
             val readReceipt = ReadReceiptEntity.where(realm, roomId, userId).findFirst()
-                              ?: return@doWithRealm
-            val readReceiptIndex = liveChunk.timelineEvents.find(readReceipt.eventId)?.root?.displayIndex
-                                   ?: Int.MIN_VALUE
-            val eventToCheckIndex = eventToCheck?.displayIndex ?: Int.MAX_VALUE
+                    ?: return@doWithRealm
+            val readReceiptIndex = liveChunk.timelineEvents.find(readReceipt.eventId)?.displayIndex
+                    ?: Int.MIN_VALUE
+            val eventToCheckIndex = eventToCheck.displayIndex
 
             eventToCheckIndex <= readReceiptIndex
         }
@@ -61,13 +61,17 @@ internal fun isReadMarkerMoreRecent(monarchy: Monarchy,
         return false
     }
     return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
-        val liveChunk = ChunkEntity.findLastLiveChunkFromRoom(realm, roomId) ?: return false
-        val eventToCheck = liveChunk.timelineEvents.find(eventId)?.root
-
+        val eventToCheck = TimelineEventEntity.where(realm, roomId = roomId, eventId = eventId).findFirst()
+        val eventToCheckChunk = eventToCheck?.chunk?.firstOrNull()
         val readMarker = ReadMarkerEntity.where(realm, roomId).findFirst() ?: return false
-        val readMarkerIndex = liveChunk.timelineEvents.find(readMarker.eventId)?.root?.displayIndex
-                              ?: Int.MIN_VALUE
-        val eventToCheckIndex = eventToCheck?.displayIndex ?: Int.MAX_VALUE
-        eventToCheckIndex <= readMarkerIndex
+        val readMarkerEvent = TimelineEventEntity.where(realm, roomId = roomId, eventId = readMarker.eventId).findFirst()
+        val readMarkerChunk = readMarkerEvent?.chunk?.firstOrNull()
+        if (eventToCheckChunk == readMarkerChunk) {
+            val readMarkerIndex = readMarkerEvent?.displayIndex ?: Int.MIN_VALUE
+            val eventToCheckIndex = eventToCheck?.displayIndex ?: Int.MAX_VALUE
+            eventToCheckIndex <= readMarkerIndex
+        } else {
+            eventToCheckChunk?.isLastForward == false
+        }
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventEntityQueries.kt

@@ -18,7 +18,6 @@ package im.vector.matrix.android.internal.database.query
 
 import im.vector.matrix.android.api.session.room.send.SendState
 import im.vector.matrix.android.internal.database.model.*
-import im.vector.matrix.android.internal.database.model.EventEntity.LinkFilterMode.*
 import io.realm.*
 import io.realm.kotlin.where
 
@@ -34,22 +33,10 @@ internal fun TimelineEventEntity.Companion.where(realm: Realm, roomId: String, e
             .`in`(TimelineEventEntityFields.EVENT_ID, eventIds.toTypedArray())
 }
 
-internal fun TimelineEventEntity.Companion.where(realm: Realm,
-                                                 roomId: String? = null,
-                                                 type: String? = null,
-                                                 linkFilterMode: EventEntity.LinkFilterMode = LINKED_ONLY): RealmQuery<TimelineEventEntity> {
-    val query = realm.where<TimelineEventEntity>()
-    if (roomId != null) {
-        query.equalTo(TimelineEventEntityFields.ROOM_ID, roomId)
-    }
-    if (type != null) {
-        query.equalTo(TimelineEventEntityFields.ROOT.TYPE, type)
-    }
-    return when (linkFilterMode) {
-        LINKED_ONLY   -> query.equalTo(TimelineEventEntityFields.ROOT.IS_UNLINKED, false)
-        UNLINKED_ONLY -> query.equalTo(TimelineEventEntityFields.ROOT.IS_UNLINKED, true)
-        BOTH          -> query
-    }
+internal fun TimelineEventEntity.Companion.whereRoomId(realm: Realm,
+                                                       roomId: String): RealmQuery<TimelineEventEntity> {
+    return realm.where<TimelineEventEntity>()
+            .equalTo(TimelineEventEntityFields.ROOM_ID, roomId)
 }
 
 internal fun TimelineEventEntity.Companion.findWithSenderMembershipEvent(realm: Realm, senderMembershipEventId: String): List<TimelineEventEntity> {
@@ -71,7 +58,7 @@ internal fun TimelineEventEntity.Companion.latestEvent(realm: Realm,
         liveEvents
     }
     return query
-            ?.sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.DESCENDING)
+            ?.sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING)
             ?.findFirst()
 }
 
@@ -83,32 +70,6 @@ internal fun RealmQuery<TimelineEventEntity>.filterTypes(filterTypes: List<Strin
     }
 }
 
-internal fun RealmQuery<TimelineEventEntity>.next(from: Int? = null, strict: Boolean = true): TimelineEventEntity? {
-    if (from != null) {
-        if (strict) {
-            this.greaterThan(TimelineEventEntityFields.ROOT.STATE_INDEX, from)
-        } else {
-            this.greaterThanOrEqualTo(TimelineEventEntityFields.ROOT.STATE_INDEX, from)
-        }
-    }
-    return this
-            .sort(TimelineEventEntityFields.ROOT.STATE_INDEX, Sort.ASCENDING)
-            .findFirst()
-}
-
-internal fun RealmQuery<TimelineEventEntity>.prev(since: Int? = null, strict: Boolean = false): TimelineEventEntity? {
-    if (since != null) {
-        if (strict) {
-            this.lessThan(TimelineEventEntityFields.ROOT.STATE_INDEX, since)
-        } else {
-            this.lessThanOrEqualTo(TimelineEventEntityFields.ROOT.STATE_INDEX, since)
-        }
-    }
-    return this
-            .sort(TimelineEventEntityFields.ROOT.STATE_INDEX, Sort.DESCENDING)
-            .findFirst()
-}
-
 internal fun RealmList<TimelineEventEntity>.find(eventId: String): TimelineEventEntity? {
     return this.where()
             .equalTo(TimelineEventEntityFields.EVENT_ID, eventId)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/di/MatrixComponent.kt

@@ -27,7 +27,6 @@ import im.vector.matrix.android.api.auth.AuthenticationService
 import im.vector.matrix.android.internal.SessionManager
 import im.vector.matrix.android.internal.auth.AuthModule
 import im.vector.matrix.android.internal.auth.SessionParamsStore
-import im.vector.matrix.android.internal.network.NetworkConnectivityChecker
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.util.BackgroundDetectionObserver
 import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers
@@ -59,8 +58,6 @@ internal interface MatrixComponent {
 
     fun sessionParamsStore(): SessionParamsStore
 
-    fun networkConnectivityChecker(): NetworkConnectivityChecker
-
     fun backgroundDetectionObserver(): BackgroundDetectionObserver
 
     fun sessionManager(): SessionManager

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/NetworkCallbackStrategy.kt

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.network
+
+import android.annotation.TargetApi
+import android.content.Context
+import android.content.IntentFilter
+import android.net.ConnectivityManager
+import android.net.Network
+import android.os.Build
+import javax.inject.Inject
+
+internal interface NetworkCallbackStrategy {
+    fun register(hasChanged: () -> Unit)
+    fun unregister()
+}
+
+internal class FallbackNetworkCallbackStrategy @Inject constructor(private val context: Context,
+                                                                   private val networkInfoReceiver: NetworkInfoReceiver) : NetworkCallbackStrategy {
+
+    @Suppress("DEPRECATION")
+    val filter = IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION)
+
+    override fun register(hasChanged: () -> Unit) {
+        networkInfoReceiver.isConnectedCallback = {
+            hasChanged()
+        }
+        context.registerReceiver(networkInfoReceiver, filter)
+    }
+
+    override fun unregister() {
+        networkInfoReceiver.isConnectedCallback = null
+        context.unregisterReceiver(networkInfoReceiver)
+    }
+}
+
+@TargetApi(Build.VERSION_CODES.N)
+internal class PreferredNetworkCallbackStrategy @Inject constructor(context: Context) : NetworkCallbackStrategy {
+
+    private var hasChangedCallback: (() -> Unit)? = null
+    private val conn = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager
+    private val networkCallback = object : ConnectivityManager.NetworkCallback() {
+
+        override fun onLost(network: Network) {
+            hasChangedCallback?.invoke()
+        }
+
+        override fun onAvailable(network: Network) {
+            hasChangedCallback?.invoke()
+        }
+    }
+
+    override fun register(hasChanged: () -> Unit) {
+        hasChangedCallback = hasChanged
+        conn.registerDefaultNetworkCallback(networkCallback)
+    }
+
+    override fun unregister() {
+        hasChangedCallback = null
+        conn.unregisterNetworkCallback(networkCallback)
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/NetworkConnectivityChecker.kt

@@ -16,112 +16,94 @@
 
 package im.vector.matrix.android.internal.network
 
-import android.content.Context
 import androidx.annotation.WorkerThread
-import com.novoda.merlin.Merlin
-import com.novoda.merlin.MerlinsBeard
-import im.vector.matrix.android.internal.di.MatrixScope
+import im.vector.matrix.android.internal.session.SessionScope
+import im.vector.matrix.android.internal.session.homeserver.HomeServerPinger
 import im.vector.matrix.android.internal.util.BackgroundDetectionObserver
-import timber.log.Timber
+import kotlinx.coroutines.runBlocking
 import java.util.Collections
+import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
-import kotlin.coroutines.resume
-import kotlin.coroutines.suspendCoroutine
 
-@MatrixScope
-internal class NetworkConnectivityChecker @Inject constructor(private val context: Context,
-                                                              private val backgroundDetectionObserver: BackgroundDetectionObserver)
-    : BackgroundDetectionObserver.Listener {
+interface NetworkConnectivityChecker {
+    /**
+     * Returns true when internet is available
+     */
+    @WorkerThread
+    fun hasInternetAccess(forcePing: Boolean): Boolean
 
-    private val merlin = Merlin.Builder()
-            .withConnectableCallbacks()
-            .withDisconnectableCallbacks()
-            .build(context)
+    fun register(listener: Listener)
+    fun unregister(listener: Listener)
 
-    private val merlinsBeard = MerlinsBeard.Builder().build(context)
+    interface Listener {
+        fun onConnectivityChanged()
+    }
+}
 
-    private val listeners = Collections.synchronizedSet(LinkedHashSet<Listener>())
-    private var hasInternetAccess = merlinsBeard.isConnected
+@SessionScope
+internal class DefaultNetworkConnectivityChecker @Inject constructor(private val homeServerPinger: HomeServerPinger,
+                                                                     private val backgroundDetectionObserver: BackgroundDetectionObserver,
+                                                                     private val networkCallbackStrategy: NetworkCallbackStrategy)
+    : NetworkConnectivityChecker {
 
-    init {
-        backgroundDetectionObserver.register(this)
+    private val hasInternetAccess = AtomicBoolean(true)
+    private val listeners = Collections.synchronizedSet(LinkedHashSet<NetworkConnectivityChecker.Listener>())
+    private val backgroundDetectionObserverListener = object : BackgroundDetectionObserver.Listener {
+        override fun onMoveToForeground() {
+            bind()
+        }
+
+        override fun onMoveToBackground() {
+            unbind()
+        }
     }
 
     /**
      * Returns true when internet is available
      */
     @WorkerThread
-    fun hasInternetAccess(): Boolean {
-        // If we are in background we have unbound merlin, so we have to check
-        return if (backgroundDetectionObserver.isInBackground) {
-            merlinsBeard.hasInternetAccess()
+    override fun hasInternetAccess(forcePing: Boolean): Boolean {
+        return if (forcePing) {
+            runBlocking {
+                homeServerPinger.canReachHomeServer()
+            }
         } else {
-            hasInternetAccess
+            hasInternetAccess.get()
         }
     }
 
-    override fun onMoveToForeground() {
-        merlin.bind()
-        merlinsBeard.hasInternetAccess {
-            hasInternetAccess = it
-        }
-        merlin.registerDisconnectable {
-            if (hasInternetAccess) {
-                Timber.v("On Disconnect")
-                hasInternetAccess = false
-                val localListeners = listeners.toList()
-                localListeners.forEach {
-                    it.onDisconnect()
-                }
+    override fun register(listener: NetworkConnectivityChecker.Listener) {
+        if (listeners.isEmpty()) {
+            if (backgroundDetectionObserver.isInBackground) {
+                unbind()
+            } else {
+                bind()
             }
+            backgroundDetectionObserver.register(backgroundDetectionObserverListener)
         }
-        merlin.registerConnectable {
-            if (!hasInternetAccess) {
-                Timber.v("On Connect")
-                hasInternetAccess = true
-                val localListeners = listeners.toList()
-                localListeners.forEach {
-                    it.onConnect()
-                }
-            }
-        }
-    }
-
-    override fun onMoveToBackground() {
-        merlin.unbind()
-    }
-
-    // In background you won't get notification as merlin is unbound
-    suspend fun waitUntilConnected() {
-        if (hasInternetAccess) {
-            return
-        } else {
-            Timber.v("Waiting for network...")
-            suspendCoroutine<Unit> { continuation ->
-                register(object : Listener {
-                    override fun onConnect() {
-                        unregister(this)
-                        Timber.v("Connected to network...")
-                        continuation.resume(Unit)
-                    }
-                })
-            }
-        }
-    }
-
-    fun register(listener: Listener) {
         listeners.add(listener)
     }
 
-    fun unregister(listener: Listener) {
+    override fun unregister(listener: NetworkConnectivityChecker.Listener) {
         listeners.remove(listener)
+        if (listeners.isEmpty()) {
+            backgroundDetectionObserver.unregister(backgroundDetectionObserverListener)
+        }
     }
 
-    interface Listener {
-        fun onConnect() {
+    private fun bind() {
+        networkCallbackStrategy.register {
+            val localListeners = listeners.toList()
+            localListeners.forEach {
+                it.onConnectivityChanged()
+            }
         }
+        homeServerPinger.canReachHomeServer {
+            hasInternetAccess.set(it)
+        }
+    }
 
-        fun onDisconnect() {
-        }
+    private fun unbind() {
+        networkCallbackStrategy.unregister()
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/NetworkInfoReceiver.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.network
+
+import android.content.BroadcastReceiver
+import android.content.Context
+import android.content.Intent
+import android.net.ConnectivityManager
+import android.net.NetworkInfo
+import javax.inject.Inject
+
+internal class NetworkInfoReceiver @Inject constructor() : BroadcastReceiver() {
+
+    var isConnectedCallback: ((Boolean) -> Unit)? = null
+
+    override fun onReceive(context: Context, intent: Intent) {
+        val conn = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager
+        val networkInfo: NetworkInfo? = conn.activeNetworkInfo
+        isConnectedCallback?.invoke(networkInfo?.isConnected ?: false)
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/network/Request.kt

@@ -18,6 +18,7 @@ package im.vector.matrix.android.internal.network
 
 import im.vector.matrix.android.api.failure.Failure
 import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.delay
 import org.greenrobot.eventbus.EventBus
 import retrofit2.Call
 import java.io.IOException
@@ -27,11 +28,17 @@ internal suspend inline fun <DATA> executeRequest(eventBus: EventBus?,
 
 internal class Request<DATA>(private val eventBus: EventBus?) {
 
+    var isRetryable = false
+    var initialDelay: Long = 100L
+    var maxDelay: Long = 10_000L
+    var maxRetryCount = Int.MAX_VALUE
+    private var currentRetryCount = 0
+    private var currentDelay = initialDelay
     lateinit var apiCall: Call<DATA>
 
     suspend fun execute(): DATA {
         return try {
-            val response = apiCall.awaitResponse()
+            val response = apiCall.clone().awaitResponse()
             if (response.isSuccessful) {
                 response.body()
                         ?: throw IllegalStateException("The request returned a null body")
@@ -39,12 +46,18 @@ internal class Request<DATA>(private val eventBus: EventBus?) {
                 throw response.toFailure(eventBus)
             }
         } catch (exception: Throwable) {
-            throw when (exception) {
-                is IOException              -> Failure.NetworkConnection(exception)
-                is Failure.ServerError,
-                is Failure.OtherServerError -> exception
-                is CancellationException    -> Failure.Cancelled(exception)
-                else                        -> Failure.Unknown(exception)
+            if (isRetryable && currentRetryCount++ < maxRetryCount && exception is IOException) {
+                delay(currentDelay)
+                currentDelay = (currentDelay * 2L).coerceAtMost(maxDelay)
+                return execute()
+            } else {
+                throw when (exception) {
+                    is IOException              -> Failure.NetworkConnection(exception)
+                    is Failure.ServerError,
+                    is Failure.OtherServerError -> exception
+                    is CancellationException    -> Failure.Cancelled(exception)
+                    else                        -> Failure.Unknown(exception)
+                }
             }
         }
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt

@@ -17,6 +17,7 @@
 package im.vector.matrix.android.internal.session
 
 import android.content.Context
+import android.os.Build
 import com.zhuinden.monarchy.Monarchy
 import dagger.Binds
 import dagger.Lazy
@@ -46,6 +47,11 @@ import im.vector.matrix.android.internal.di.Unauthenticated
 import im.vector.matrix.android.internal.di.UserId
 import im.vector.matrix.android.internal.di.UserMd5
 import im.vector.matrix.android.internal.network.AccessTokenInterceptor
+import im.vector.matrix.android.internal.network.DefaultNetworkConnectivityChecker
+import im.vector.matrix.android.internal.network.FallbackNetworkCallbackStrategy
+import im.vector.matrix.android.internal.network.NetworkCallbackStrategy
+import im.vector.matrix.android.internal.network.NetworkConnectivityChecker
+import im.vector.matrix.android.internal.network.PreferredNetworkCallbackStrategy
 import im.vector.matrix.android.internal.network.RetrofitFactory
 import im.vector.matrix.android.internal.network.interceptors.CurlLoggingInterceptor
 import im.vector.matrix.android.internal.session.group.GroupSummaryUpdater
@@ -61,6 +67,7 @@ import okhttp3.OkHttpClient
 import org.greenrobot.eventbus.EventBus
 import retrofit2.Retrofit
 import java.io.File
+import javax.inject.Provider
 
 @Module
 internal abstract class SessionModule {
@@ -69,6 +76,11 @@ internal abstract class SessionModule {
     companion object {
         internal fun getKeyAlias(userMd5: String) = "session_db_$userMd5"
 
+        /**
+         * Rules:
+         * Annotate methods with @SessionScope only the @Provides annotated methods with computation and logic.
+         */
+
         @JvmStatic
         @Provides
         fun providesHomeServerConnectionConfig(sessionParams: SessionParams): HomeServerConnectionConfig {
@@ -84,6 +96,7 @@ internal abstract class SessionModule {
         @JvmStatic
         @UserId
         @Provides
+        @SessionScope
         fun providesUserId(credentials: Credentials): String {
             return credentials.userId
         }
@@ -98,6 +111,7 @@ internal abstract class SessionModule {
         @JvmStatic
         @UserMd5
         @Provides
+        @SessionScope
         fun providesUserMd5(@UserId userId: String): String {
             return userId.md5()
         }
@@ -105,6 +119,7 @@ internal abstract class SessionModule {
         @JvmStatic
         @SessionId
         @Provides
+        @SessionScope
         fun providesSessionId(credentials: Credentials): String {
             return credentials.sessionId()
         }
@@ -189,6 +204,19 @@ internal abstract class SessionModule {
             return EventBus.builder().build()
         }
 
+        @JvmStatic
+        @Provides
+        @SessionScope
+        fun providesNetworkCallbackStrategy(fallbackNetworkCallbackStrategy: Provider<FallbackNetworkCallbackStrategy>,
+                                            preferredNetworkCallbackStrategy: Provider<PreferredNetworkCallbackStrategy>
+        ): NetworkCallbackStrategy {
+            return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+                preferredNetworkCallbackStrategy.get()
+            } else {
+                fallbackNetworkCallbackStrategy.get()
+            }
+        }
+
         @JvmStatic
         @Provides
         @SessionScope
@@ -200,6 +228,9 @@ internal abstract class SessionModule {
     @Binds
     abstract fun bindSession(session: DefaultSession): Session
 
+    @Binds
+    abstract fun bindNetworkConnectivityChecker(networkConnectivityChecker: DefaultNetworkConnectivityChecker): NetworkConnectivityChecker
+
     @Binds
     @IntoSet
     abstract fun bindGroupSummaryUpdater(groupSummaryUpdater: GroupSummaryUpdater): LiveEntityObserver

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/group/DefaultGetGroupDataTask.kt

@@ -25,6 +25,7 @@ import im.vector.matrix.android.internal.session.group.model.GroupRooms
 import im.vector.matrix.android.internal.session.group.model.GroupSummaryResponse
 import im.vector.matrix.android.internal.session.group.model.GroupUsers
 import im.vector.matrix.android.internal.task.Task
+import im.vector.matrix.android.internal.util.awaitTransaction
 import org.greenrobot.eventbus.EventBus
 import javax.inject.Inject
 
@@ -53,12 +54,12 @@ internal class DefaultGetGroupDataTask @Inject constructor(
         insertInDb(groupSummary, groupRooms, groupUsers, groupId)
     }
 
-    private fun insertInDb(groupSummary: GroupSummaryResponse,
+    private suspend fun insertInDb(groupSummary: GroupSummaryResponse,
                            groupRooms: GroupRooms,
                            groupUsers: GroupUsers,
                            groupId: String) {
         monarchy
-                .writeAsync { realm ->
+                .awaitTransaction { realm ->
                     val groupSummaryEntity = GroupSummaryEntity.where(realm, groupId).findFirst()
                             ?: realm.createObject(GroupSummaryEntity::class.java, groupId)
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/homeserver/CapabilitiesAPI.kt

@@ -27,4 +27,10 @@ internal interface CapabilitiesAPI {
      */
     @GET(NetworkConstants.URI_API_MEDIA_PREFIX_PATH_R0 + "config")
     fun getUploadCapabilities(): Call<GetUploadCapabilitiesResult>
+
+    /**
+     * Request the versions
+     */
+    @GET(NetworkConstants.URI_API_PREFIX_PATH_ + "versions")
+    fun getVersions(): Call<Unit>
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/homeserver/HomeServerPinger.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.session.homeserver
+
+import im.vector.matrix.android.api.failure.Failure
+import im.vector.matrix.android.internal.network.executeRequest
+import im.vector.matrix.android.internal.task.TaskExecutor
+import kotlinx.coroutines.launch
+import javax.inject.Inject
+
+internal class HomeServerPinger @Inject constructor(private val taskExecutor: TaskExecutor,
+                                                    private val capabilitiesAPI: CapabilitiesAPI) {
+
+    fun canReachHomeServer(callback: (Boolean) -> Unit) {
+        taskExecutor.executorScope.launch {
+            val canReach = canReachHomeServer()
+            callback(canReach)
+        }
+    }
+
+    suspend fun canReachHomeServer(): Boolean {
+        return try {
+            executeRequest<Unit>(null) {
+                apiCall = capabilitiesAPI.getVersions()
+            }
+            true
+        } catch (throwable: Throwable) {
+            if (throwable is Failure.OtherServerError) {
+                (throwable.httpCode == 404 || throwable.httpCode == 400)
+            } else {
+                false
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/EventRelationsAggregationUpdater.kt

@@ -20,7 +20,7 @@ import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.di.SessionDatabase
 import im.vector.matrix.android.internal.di.UserId
 import io.realm.OrderedCollectionChangeSet
@@ -42,7 +42,7 @@ internal class EventRelationsAggregationUpdater @Inject constructor(
         RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(
+        EventEntity.whereTypes(it, listOf(
                 EventType.MESSAGE,
                 EventType.REDACTION,
                 EventType.REACTION,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomAvatarResolver.kt

@@ -21,10 +21,9 @@ import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.events.model.toModel
 import im.vector.matrix.android.api.session.room.model.RoomAvatarContent
 import im.vector.matrix.android.internal.database.mapper.ContentMapper
-import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
-import im.vector.matrix.android.internal.database.query.prev
-import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.di.UserId
 import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
 import javax.inject.Inject
@@ -40,7 +39,7 @@ internal class RoomAvatarResolver @Inject constructor(private val monarchy: Mona
     fun resolve(roomId: String): String? {
         var res: String? = null
         monarchy.doWithRealm { realm ->
-            val roomName = EventEntity.where(realm, roomId, EventType.STATE_ROOM_AVATAR).prev()
+            val roomName = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_AVATAR, stateKey = "")?.root
             res = ContentMapper.map(roomName?.content).toModel<RoomAvatarContent>()?.avatarUrl
             if (!res.isNullOrEmpty()) {
                 return@doWithRealm

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomFactory.kt

@@ -59,22 +59,22 @@ internal class DefaultRoomFactory @Inject constructor(private val monarchy: Mona
 
     override fun create(roomId: String): Room {
         return DefaultRoom(
-                roomId,
-                monarchy,
-                roomSummaryMapper,
-                timelineServiceFactory.create(roomId),
-                sendServiceFactory.create(roomId),
-                draftServiceFactory.create(roomId),
-                stateServiceFactory.create(roomId),
-                reportingServiceFactory.create(roomId),
-                readServiceFactory.create(roomId),
-                typingServiceFactory.create(roomId),
-                cryptoService,
-                relationServiceFactory.create(roomId),
-                membershipServiceFactory.create(roomId),
-                roomPushRuleServiceFactory.create(roomId),
-                taskExecutor,
-                sendStateTask
+                roomId = roomId,
+                monarchy = monarchy,
+                roomSummaryMapper = roomSummaryMapper,
+                timelineService = timelineServiceFactory.create(roomId),
+                sendService = sendServiceFactory.create(roomId),
+                draftService = draftServiceFactory.create(roomId),
+                stateService = stateServiceFactory.create(roomId),
+                reportingService = reportingServiceFactory.create(roomId),
+                readService = readServiceFactory.create(roomId),
+                typingService = typingServiceFactory.create(roomId),
+                cryptoService = cryptoService,
+                relationService = relationServiceFactory.create(roomId),
+                roomMembersService = membershipServiceFactory.create(roomId),
+                roomPushRuleService = roomPushRuleServiceFactory.create(roomId),
+                taskExecutor = taskExecutor,
+                sendStateTask = sendStateTask
         )
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt

@@ -129,9 +129,6 @@ internal abstract class RoomModule {
     @Binds
     abstract fun bindGetContextOfEventTask(task: DefaultGetContextOfEventTask): GetContextOfEventTask
 
-    @Binds
-    abstract fun bindClearUnlinkedEventsTask(task: DefaultClearUnlinkedEventsTask): ClearUnlinkedEventsTask
-
     @Binds
     abstract fun bindPaginationTask(task: DefaultPaginationTask): PaginationTask
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomSummaryUpdater.kt

@@ -25,16 +25,17 @@ import im.vector.matrix.android.api.session.room.model.RoomCanonicalAliasContent
 import im.vector.matrix.android.api.session.room.model.RoomTopicContent
 import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import im.vector.matrix.android.internal.database.mapper.ContentMapper
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
 import im.vector.matrix.android.internal.database.model.TimelineEventEntity
 import im.vector.matrix.android.internal.database.query.getOrCreate
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.database.query.isEventRead
 import im.vector.matrix.android.internal.database.query.latestEvent
-import im.vector.matrix.android.internal.database.query.prev
-import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.database.query.whereType
 import im.vector.matrix.android.internal.di.UserId
 import im.vector.matrix.android.internal.session.room.membership.RoomDisplayNameResolver
 import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
@@ -50,23 +51,25 @@ internal class RoomSummaryUpdater @Inject constructor(
         private val roomAvatarResolver: RoomAvatarResolver,
         private val monarchy: Monarchy) {
 
-    // TODO: maybe allow user of SDK to give that list
-    private val PREVIEWABLE_TYPES = listOf(
-            // TODO filter message type (KEY_VERIFICATION_READY, etc.)
-            EventType.MESSAGE,
-            EventType.STATE_ROOM_NAME,
-            EventType.STATE_ROOM_TOPIC,
-            EventType.STATE_ROOM_MEMBER,
-            EventType.STATE_ROOM_HISTORY_VISIBILITY,
-            EventType.CALL_INVITE,
-            EventType.CALL_HANGUP,
-            EventType.CALL_ANSWER,
-            EventType.ENCRYPTED,
-            EventType.STATE_ROOM_ENCRYPTION,
-            EventType.STATE_ROOM_THIRD_PARTY_INVITE,
-            EventType.STICKER,
-            EventType.STATE_ROOM_CREATE
-    )
+    companion object {
+        // TODO: maybe allow user of SDK to give that list
+        val PREVIEWABLE_TYPES = listOf(
+                // TODO filter message type (KEY_VERIFICATION_READY, etc.)
+                EventType.MESSAGE,
+                EventType.STATE_ROOM_NAME,
+                EventType.STATE_ROOM_TOPIC,
+                EventType.STATE_ROOM_MEMBER,
+                EventType.STATE_ROOM_HISTORY_VISIBILITY,
+                EventType.CALL_INVITE,
+                EventType.CALL_HANGUP,
+                EventType.CALL_ANSWER,
+                EventType.ENCRYPTED,
+                EventType.STATE_ROOM_ENCRYPTION,
+                EventType.STATE_ROOM_THIRD_PARTY_INVITE,
+                EventType.STICKER,
+                EventType.STATE_ROOM_CREATE
+        )
+    }
 
     fun update(realm: Realm,
                roomId: String,
@@ -96,12 +99,15 @@ internal class RoomSummaryUpdater @Inject constructor(
         }
 
         val latestPreviewableEvent = TimelineEventEntity.latestEvent(realm, roomId, includesSending = true, filterTypes = PREVIEWABLE_TYPES)
-        val lastTopicEvent = EventEntity.where(realm, roomId, EventType.STATE_ROOM_TOPIC).prev()
-        val lastCanonicalAliasEvent = EventEntity.where(realm, roomId, EventType.STATE_ROOM_CANONICAL_ALIAS).prev()
-        val lastAliasesEvent = EventEntity.where(realm, roomId, EventType.STATE_ROOM_ALIASES).prev()
-        val encryptionEvent = EventEntity.where(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
+
+        val lastTopicEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_TOPIC, stateKey = "")?.root
+        val lastCanonicalAliasEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_CANONICAL_ALIAS, stateKey = "")?.root
+        val lastAliasesEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_ALIASES, stateKey = "")?.root
+
+        // Don't use current state for this one as we are only interested in having MXCRYPTO_ALGORITHM_MEGOLM event in the room
+        val encryptionEvent = EventEntity.whereType(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
                 .contains(EventEntityFields.CONTENT, "\"algorithm\":\"$MXCRYPTO_ALGORITHM_MEGOLM\"")
-                .prev()
+                .findFirst()
 
         roomSummaryEntity.hasUnreadMessages = roomSummaryEntity.notificationCount > 0
                 // avoid this call if we are sure there are unread events

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/create/CreateRoomTask.kt

@@ -120,7 +120,7 @@ internal class DefaultCreateRoomTask @Inject constructor(
     }
 
     private suspend fun setReadMarkers(roomId: String) {
-        val setReadMarkerParams = SetReadMarkersTask.Params(roomId, markAllAsRead = true)
+        val setReadMarkerParams = SetReadMarkersTask.Params(roomId, forceReadReceipt = true, forceReadMarker = true)
         return readMarkersTask.execute(setReadMarkerParams)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/create/RoomCreateEventLiveObserver.kt

@@ -27,7 +27,7 @@ import im.vector.matrix.android.internal.database.awaitTransaction
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
@@ -41,7 +41,7 @@ internal class RoomCreateEventLiveObserver @Inject constructor(@SessionDatabase
     : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(EventType.STATE_ROOM_CREATE))
+        EventEntity.whereTypes(it, listOf(EventType.STATE_ROOM_CREATE))
     }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/draft/DefaultDraftService.kt

@@ -17,23 +17,20 @@
 package im.vector.matrix.android.internal.session.room.draft
 
 import androidx.lifecycle.LiveData
-import androidx.lifecycle.Transformations
 import com.squareup.inject.assisted.Assisted
 import com.squareup.inject.assisted.AssistedInject
-import com.zhuinden.monarchy.Monarchy
-import im.vector.matrix.android.BuildConfig
+import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.session.room.send.DraftService
 import im.vector.matrix.android.api.session.room.send.UserDraft
-import im.vector.matrix.android.internal.database.mapper.DraftMapper
-import im.vector.matrix.android.internal.database.model.DraftEntity
-import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.model.UserDraftsEntity
-import im.vector.matrix.android.internal.database.query.where
-import io.realm.kotlin.createObject
-import timber.log.Timber
+import im.vector.matrix.android.api.util.Cancelable
+import im.vector.matrix.android.internal.task.TaskExecutor
+import im.vector.matrix.android.internal.task.launchToCallback
+import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers
 
 internal class DefaultDraftService @AssistedInject constructor(@Assisted private val roomId: String,
-                                                               private val monarchy: Monarchy
+                                                               private val draftRepository: DraftRepository,
+                                                               private val taskExecutor: TaskExecutor,
+                                                               private val coroutineDispatchers: MatrixCoroutineDispatchers
 ) : DraftService {
 
     @AssistedInject.Factory
@@ -45,121 +42,19 @@ internal class DefaultDraftService @AssistedInject constructor(@Assisted private
      * The draft stack can contain several drafts. Depending of the draft to save, it will update the top draft, or create a new draft,
      * or even move an existing draft to the top of the list
      */
-    override fun saveDraft(draft: UserDraft) {
-        Timber.d("Draft: saveDraft ${privacySafe(draft)}")
-
-        monarchy.writeAsync { realm ->
-
-            val roomSummaryEntity = RoomSummaryEntity.where(realm, roomId).findFirst()
-                                    ?: realm.createObject(roomId)
-
-            val userDraftsEntity = roomSummaryEntity.userDrafts
-                                   ?: realm.createObject<UserDraftsEntity>().also {
-                                       roomSummaryEntity.userDrafts = it
-                                   }
-
-            userDraftsEntity.let { userDraftEntity ->
-                // Save only valid draft
-                if (draft.isValid()) {
-                    // Add a new draft or update the current one?
-                    val newDraft = DraftMapper.map(draft)
-
-                    // Is it an update of the top draft?
-                    val topDraft = userDraftEntity.userDrafts.lastOrNull()
-
-                    if (topDraft == null) {
-                        Timber.d("Draft: create a new draft ${privacySafe(draft)}")
-                        userDraftEntity.userDrafts.add(newDraft)
-                    } else if (topDraft.draftMode == DraftEntity.MODE_EDIT) {
-                        // top draft is an edit
-                        if (newDraft.draftMode == DraftEntity.MODE_EDIT) {
-                            if (topDraft.linkedEventId == newDraft.linkedEventId) {
-                                // Update the top draft
-                                Timber.d("Draft: update the top edit draft ${privacySafe(draft)}")
-                                topDraft.content = newDraft.content
-                            } else {
-                                // Check a previously EDIT draft with the same id
-                                val existingEditDraftOfSameEvent = userDraftEntity.userDrafts.find {
-                                    it.draftMode == DraftEntity.MODE_EDIT && it.linkedEventId == newDraft.linkedEventId
-                                }
-
-                                if (existingEditDraftOfSameEvent != null) {
-                                    // Ignore the new text, restore what was typed before, by putting the draft to the top
-                                    Timber.d("Draft: restore a previously edit draft ${privacySafe(draft)}")
-                                    userDraftEntity.userDrafts.remove(existingEditDraftOfSameEvent)
-                                    userDraftEntity.userDrafts.add(existingEditDraftOfSameEvent)
-                                } else {
-                                    Timber.d("Draft: add a new edit draft ${privacySafe(draft)}")
-                                    userDraftEntity.userDrafts.add(newDraft)
-                                }
-                            }
-                        } else {
-                            // Add a new regular draft to the top
-                            Timber.d("Draft: add a new draft ${privacySafe(draft)}")
-                            userDraftEntity.userDrafts.add(newDraft)
-                        }
-                    } else {
-                        // Top draft is not an edit
-                        if (newDraft.draftMode == DraftEntity.MODE_EDIT) {
-                            Timber.d("Draft: create a new edit draft ${privacySafe(draft)}")
-                            userDraftEntity.userDrafts.add(newDraft)
-                        } else {
-                            // Update the top draft
-                            Timber.d("Draft: update the top draft ${privacySafe(draft)}")
-                            topDraft.draftMode = newDraft.draftMode
-                            topDraft.content = newDraft.content
-                            topDraft.linkedEventId = newDraft.linkedEventId
-                        }
-                    }
-                } else {
-                    // There is no draft to save, so the composer was clear
-                    Timber.d("Draft: delete a draft")
-
-                    val topDraft = userDraftEntity.userDrafts.lastOrNull()
-
-                    if (topDraft == null) {
-                        Timber.d("Draft: nothing to do")
-                    } else {
-                        // Remove the top draft
-                        Timber.d("Draft: remove the top draft")
-                        userDraftEntity.userDrafts.remove(topDraft)
-                    }
-                }
-            }
+    override fun saveDraft(draft: UserDraft, callback: MatrixCallback<Unit>): Cancelable {
+        return taskExecutor.executorScope.launchToCallback(coroutineDispatchers.main, callback) {
+            draftRepository.saveDraft(roomId, draft)
         }
     }
 
-    private fun privacySafe(o: Any): Any {
-        if (BuildConfig.LOG_PRIVATE_DATA) {
-            return o
-        }
-
-        return ""
-    }
-
-    override fun deleteDraft() {
-        Timber.d("Draft: deleteDraft()")
-
-        monarchy.writeAsync { realm ->
-            UserDraftsEntity.where(realm, roomId).findFirst()?.let { userDraftsEntity ->
-                if (userDraftsEntity.userDrafts.isNotEmpty()) {
-                    userDraftsEntity.userDrafts.removeAt(userDraftsEntity.userDrafts.size - 1)
-                }
-            }
+    override fun deleteDraft(callback: MatrixCallback<Unit>): Cancelable {
+        return taskExecutor.executorScope.launchToCallback(coroutineDispatchers.main, callback) {
+            draftRepository.deleteDraft(roomId)
         }
     }
 
     override fun getDraftsLive(): LiveData<List<UserDraft>> {
-        val liveData = monarchy.findAllMappedWithChanges(
-                { UserDraftsEntity.where(it, roomId) },
-                {
-                    it.userDrafts.map { draft ->
-                        DraftMapper.map(draft)
-                    }
-                }
-        )
-        return Transformations.map(liveData) {
-            it.firstOrNull() ?: emptyList()
-        }
+        return draftRepository.getDraftsLive(roomId)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/draft/DraftRepository.kt

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.session.room.draft
+
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.Transformations
+import com.zhuinden.monarchy.Monarchy
+import im.vector.matrix.android.BuildConfig
+import im.vector.matrix.android.api.session.room.send.UserDraft
+import im.vector.matrix.android.internal.database.mapper.DraftMapper
+import im.vector.matrix.android.internal.database.model.DraftEntity
+import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
+import im.vector.matrix.android.internal.database.model.UserDraftsEntity
+import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.util.awaitTransaction
+import io.realm.Realm
+import io.realm.kotlin.createObject
+import timber.log.Timber
+import javax.inject.Inject
+
+class DraftRepository @Inject constructor(private val monarchy: Monarchy) {
+
+    suspend fun saveDraft(roomId: String, userDraft: UserDraft) {
+        monarchy.awaitTransaction {
+            saveDraft(it, userDraft, roomId)
+        }
+    }
+
+    suspend fun deleteDraft(roomId: String) {
+        monarchy.awaitTransaction {
+            deleteDraft(it, roomId)
+        }
+    }
+
+    private fun deleteDraft(realm: Realm, roomId: String) {
+        UserDraftsEntity.where(realm, roomId).findFirst()?.let { userDraftsEntity ->
+            if (userDraftsEntity.userDrafts.isNotEmpty()) {
+                userDraftsEntity.userDrafts.removeAt(userDraftsEntity.userDrafts.size - 1)
+            }
+        }
+    }
+
+    private fun saveDraft(realm: Realm, draft: UserDraft, roomId: String) {
+        val roomSummaryEntity = RoomSummaryEntity.where(realm, roomId).findFirst()
+                ?: realm.createObject(roomId)
+
+        val userDraftsEntity = roomSummaryEntity.userDrafts
+                ?: realm.createObject<UserDraftsEntity>().also {
+                    roomSummaryEntity.userDrafts = it
+                }
+
+        userDraftsEntity.let { userDraftEntity ->
+            // Save only valid draft
+            if (draft.isValid()) {
+                // Add a new draft or update the current one?
+                val newDraft = DraftMapper.map(draft)
+
+                // Is it an update of the top draft?
+                val topDraft = userDraftEntity.userDrafts.lastOrNull()
+
+                if (topDraft == null) {
+                    Timber.d("Draft: create a new draft ${privacySafe(draft)}")
+                    userDraftEntity.userDrafts.add(newDraft)
+                } else if (topDraft.draftMode == DraftEntity.MODE_EDIT) {
+                    // top draft is an edit
+                    if (newDraft.draftMode == DraftEntity.MODE_EDIT) {
+                        if (topDraft.linkedEventId == newDraft.linkedEventId) {
+                            // Update the top draft
+                            Timber.d("Draft: update the top edit draft ${privacySafe(draft)}")
+                            topDraft.content = newDraft.content
+                        } else {
+                            // Check a previously EDIT draft with the same id
+                            val existingEditDraftOfSameEvent = userDraftEntity.userDrafts.find {
+                                it.draftMode == DraftEntity.MODE_EDIT && it.linkedEventId == newDraft.linkedEventId
+                            }
+
+                            if (existingEditDraftOfSameEvent != null) {
+                                // Ignore the new text, restore what was typed before, by putting the draft to the top
+                                Timber.d("Draft: restore a previously edit draft ${privacySafe(draft)}")
+                                userDraftEntity.userDrafts.remove(existingEditDraftOfSameEvent)
+                                userDraftEntity.userDrafts.add(existingEditDraftOfSameEvent)
+                            } else {
+                                Timber.d("Draft: add a new edit draft ${privacySafe(draft)}")
+                                userDraftEntity.userDrafts.add(newDraft)
+                            }
+                        }
+                    } else {
+                        // Add a new regular draft to the top
+                        Timber.d("Draft: add a new draft ${privacySafe(draft)}")
+                        userDraftEntity.userDrafts.add(newDraft)
+                    }
+                } else {
+                    // Top draft is not an edit
+                    if (newDraft.draftMode == DraftEntity.MODE_EDIT) {
+                        Timber.d("Draft: create a new edit draft ${privacySafe(draft)}")
+                        userDraftEntity.userDrafts.add(newDraft)
+                    } else {
+                        // Update the top draft
+                        Timber.d("Draft: update the top draft ${privacySafe(draft)}")
+                        topDraft.draftMode = newDraft.draftMode
+                        topDraft.content = newDraft.content
+                        topDraft.linkedEventId = newDraft.linkedEventId
+                    }
+                }
+            } else {
+                // There is no draft to save, so the composer was clear
+                Timber.d("Draft: delete a draft")
+
+                val topDraft = userDraftEntity.userDrafts.lastOrNull()
+
+                if (topDraft == null) {
+                    Timber.d("Draft: nothing to do")
+                } else {
+                    // Remove the top draft
+                    Timber.d("Draft: remove the top draft")
+                    userDraftEntity.userDrafts.remove(topDraft)
+                }
+            }
+        }
+    }
+
+    fun getDraftsLive(roomId: String): LiveData<List<UserDraft>> {
+        val liveData = monarchy.findAllMappedWithChanges(
+                { UserDraftsEntity.where(it, roomId) },
+                {
+                    it.userDrafts.map { draft ->
+                        DraftMapper.map(draft)
+                    }
+                }
+        )
+        return Transformations.map(liveData) {
+            it.firstOrNull() ?: emptyList()
+        }
+    }
+
+    private fun privacySafe(o: Any): Any {
+        if (BuildConfig.LOG_PRIVATE_DATA) {
+            return o
+        }
+        return ""
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/membership/LoadRoomMembersTask.kt

@@ -18,9 +18,11 @@ package im.vector.matrix.android.internal.session.room.membership
 
 import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.api.session.room.model.Membership
-import im.vector.matrix.android.internal.database.helper.TimelineEventSenderVisitor
-import im.vector.matrix.android.internal.database.helper.addStateEvent
+import im.vector.matrix.android.api.session.room.send.SendState
+import im.vector.matrix.android.internal.database.mapper.toEntity
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.query.getOrCreate
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.network.executeRequest
 import im.vector.matrix.android.internal.session.room.RoomAPI
@@ -47,7 +49,6 @@ internal class DefaultLoadRoomMembersTask @Inject constructor(
         private val syncTokenStore: SyncTokenStore,
         private val roomSummaryUpdater: RoomSummaryUpdater,
         private val roomMemberEventHandler: RoomMemberEventHandler,
-        private val timelineEventSenderVisitor: TimelineEventSenderVisitor,
         private val eventBus: EventBus
 ) : LoadRoomMembersTask {
 
@@ -69,13 +70,18 @@ internal class DefaultLoadRoomMembersTask @Inject constructor(
                     ?: realm.createObject(roomId)
 
             for (roomMemberEvent in response.roomMemberEvents) {
-                roomEntity.addStateEvent(roomMemberEvent)
+                if (roomMemberEvent.eventId == null || roomMemberEvent.stateKey == null) {
+                    continue
+                }
+                val eventEntity = roomMemberEvent.toEntity(roomId, SendState.SYNCED).let {
+                    realm.copyToRealmOrUpdate(it)
+                }
+                CurrentStateEventEntity.getOrCreate(realm, roomId, roomMemberEvent.stateKey, roomMemberEvent.type).apply {
+                    eventId = roomMemberEvent.eventId
+                    root = eventEntity
+                }
                 roomMemberEventHandler.handle(realm, roomId, roomMemberEvent)
             }
-            timelineEventSenderVisitor.clear()
-            roomEntity.chunks.flatMap { it.timelineEvents }.forEach {
-                timelineEventSenderVisitor.visit(it)
-            }
             roomEntity.areAllMembersLoaded = true
             roomSummaryUpdater.update(realm, roomId, updateMembers = true)
         }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/membership/RoomDisplayNameResolver.kt

@@ -21,14 +21,17 @@ import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.R
 import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.events.model.toModel
-import im.vector.matrix.android.api.session.room.model.*
+import im.vector.matrix.android.api.session.room.model.Membership
+import im.vector.matrix.android.api.session.room.model.RoomAliasesContent
+import im.vector.matrix.android.api.session.room.model.RoomCanonicalAliasContent
+import im.vector.matrix.android.api.session.room.model.RoomNameContent
 import im.vector.matrix.android.internal.database.mapper.ContentMapper
-import im.vector.matrix.android.internal.database.model.*
-import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.query.prev
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.UserId
 import javax.inject.Inject
@@ -57,19 +60,19 @@ internal class RoomDisplayNameResolver @Inject constructor(private val context:
         var name: CharSequence? = null
         monarchy.doWithRealm { realm ->
             val roomEntity = RoomEntity.where(realm, roomId = roomId).findFirst()
-            val roomName = EventEntity.where(realm, roomId, EventType.STATE_ROOM_NAME).prev()
+            val roomName = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_NAME, stateKey = "")?.root
             name = ContentMapper.map(roomName?.content).toModel<RoomNameContent>()?.name
             if (!name.isNullOrEmpty()) {
                 return@doWithRealm
             }
 
-            val canonicalAlias = EventEntity.where(realm, roomId, EventType.STATE_ROOM_CANONICAL_ALIAS).prev()
+            val canonicalAlias = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_CANONICAL_ALIAS, stateKey = "")?.root
             name = ContentMapper.map(canonicalAlias?.content).toModel<RoomCanonicalAliasContent>()?.canonicalAlias
             if (!name.isNullOrEmpty()) {
                 return@doWithRealm
             }
 
-            val aliases = EventEntity.where(realm, roomId, EventType.STATE_ROOM_ALIASES).prev()
+            val aliases = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_ALIASES, stateKey = "")?.root
             name = ContentMapper.map(aliases?.content).toModel<RoomAliasesContent>()?.aliases?.firstOrNull()
             if (!name.isNullOrEmpty()) {
                 return@doWithRealm

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/membership/RoomMemberHelper.kt

@@ -18,14 +18,15 @@ package im.vector.matrix.android.internal.session.room.membership
 
 import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.room.model.Membership
-import im.vector.matrix.android.internal.database.model.*
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.database.query.where
 import io.realm.Realm
 import io.realm.RealmQuery
-import io.realm.Sort
 
 /**
  * This class is an helper around STATE_ROOM_MEMBER events.
@@ -41,11 +42,7 @@ internal class RoomMemberHelper(private val realm: Realm,
     }
 
     fun getLastStateEvent(userId: String): EventEntity? {
-        return EventEntity
-                .where(realm, roomId, EventType.STATE_ROOM_MEMBER)
-                .equalTo(EventEntityFields.STATE_KEY, userId)
-                .sort(EventEntityFields.STATE_INDEX, Sort.DESCENDING)
-                .findFirst()
+        return CurrentStateEventEntity.getOrNull(realm, roomId, userId, EventType.STATE_ROOM_MEMBER)?.root
     }
 
     fun getLastRoomMember(userId: String): RoomMemberSummaryEntity? {
@@ -58,10 +55,8 @@ internal class RoomMemberHelper(private val realm: Realm,
         if (displayName.isNullOrEmpty()) {
             return true
         }
-        return EventEntity
-                .where(realm, roomId, EventType.STATE_ROOM_MEMBER)
-                .contains(EventEntityFields.CONTENT, "\"displayname\":\"$displayName\"")
-                .distinct(EventEntityFields.STATE_KEY)
+        return RoomMemberSummaryEntity.where(realm, roomId)
+                .equalTo(RoomMemberSummaryEntityFields.DISPLAY_NAME, displayName)
                 .findAll()
                 .size == 1
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/membership/joining/JoinRoomTask.kt

@@ -64,7 +64,7 @@ internal class DefaultJoinRoomTask @Inject constructor(
     }
 
     private suspend fun setReadMarkers(roomId: String) {
-        val setReadMarkerParams = SetReadMarkersTask.Params(roomId, markAllAsRead = true)
+        val setReadMarkerParams = SetReadMarkersTask.Params(roomId, forceReadMarker = true, forceReadReceipt = true)
         readMarkersTask.execute(setReadMarkerParams)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/prune/EventsPruner.kt

@@ -21,7 +21,7 @@ import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
 import io.realm.RealmConfiguration
@@ -38,7 +38,7 @@ internal class EventsPruner @Inject constructor(@SessionDatabase realmConfigurat
                                                 private val pruneEventTask: PruneEventTask) :
         RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
-    override val query = Monarchy.Query<EventEntity> { EventEntity.types(it, listOf(EventType.REDACTION)) }
+    override val query = Monarchy.Query<EventEntity> { EventEntity.whereTypes(it, listOf(EventType.REDACTION)) }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {
         Timber.v("Event pruner called with ${changeSet.insertions.size} insertions")

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/prune/PruneEventTask.kt

@@ -20,7 +20,6 @@ import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.events.model.LocalEcho
 import im.vector.matrix.android.api.session.events.model.UnsignedData
-import im.vector.matrix.android.internal.database.helper.TimelineEventSenderVisitor
 import im.vector.matrix.android.internal.database.mapper.ContentMapper
 import im.vector.matrix.android.internal.database.mapper.EventMapper
 import im.vector.matrix.android.internal.database.model.EventEntity
@@ -41,8 +40,7 @@ internal interface PruneEventTask : Task<PruneEventTask.Params, Unit> {
     )
 }
 
-internal class DefaultPruneEventTask @Inject constructor(private val monarchy: Monarchy,
-                                                         private val timelineEventSenderVisitor: TimelineEventSenderVisitor) : PruneEventTask {
+internal class DefaultPruneEventTask @Inject constructor(private val monarchy: Monarchy) : PruneEventTask {
 
     override suspend fun execute(params: PruneEventTask.Params) {
         monarchy.awaitTransaction { realm ->
@@ -98,9 +96,11 @@ internal class DefaultPruneEventTask @Inject constructor(private val monarchy: M
             }
         }
         if (typeToPrune == EventType.STATE_ROOM_MEMBER && stateKey != null) {
-            timelineEventSenderVisitor.clear(roomId = eventToPrune.roomId, senderId = stateKey)
-            val timelineEventsToUpdate = TimelineEventEntity.findWithSenderMembershipEvent(realm, eventToPrune.eventId)
-            timelineEventSenderVisitor.visit(timelineEventsToUpdate)
+            TimelineEventEntity.findWithSenderMembershipEvent(realm, eventToPrune.eventId).forEach {
+                it.senderName = null
+                it.isUniqueDisplayName = false
+                it.senderAvatar = null
+            }
         }
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/read/DefaultReadService.kt

@@ -50,10 +50,14 @@ internal class DefaultReadService @AssistedInject constructor(
         fun create(roomId: String): ReadService
     }
 
-    override fun markAllAsRead(callback: MatrixCallback<Unit>) {
-        val params = SetReadMarkersTask.Params(roomId, markAllAsRead = true)
+    override fun markAsRead(params: ReadService.MarkAsReadParams, callback: MatrixCallback<Unit>) {
+        val taskParams = SetReadMarkersTask.Params(
+                roomId = roomId,
+                forceReadMarker = params.forceReadMarker(),
+                forceReadReceipt = params.forceReadReceipt()
+        )
         setReadMarkersTask
-                .configureWith(params) {
+                .configureWith(taskParams) {
                     this.callback = callback
                 }
                 .executeBy(taskExecutor)
@@ -110,4 +114,12 @@ internal class DefaultReadService @AssistedInject constructor(
             it.firstOrNull() ?: emptyList()
         }
     }
+
+    private fun ReadService.MarkAsReadParams.forceReadMarker(): Boolean {
+        return this == ReadService.MarkAsReadParams.READ_MARKER || this == ReadService.MarkAsReadParams.BOTH
+    }
+
+    private fun ReadService.MarkAsReadParams.forceReadReceipt(): Boolean {
+        return this == ReadService.MarkAsReadParams.READ_RECEIPT || this == ReadService.MarkAsReadParams.BOTH
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/read/MarkAllRoomsReadTask.kt

@@ -29,7 +29,7 @@ internal class DefaultMarkAllRoomsReadTask @Inject constructor(private val readM
 
     override suspend fun execute(params: MarkAllRoomsReadTask.Params) {
         params.roomIds.forEach { roomId ->
-            readMarkersTask.execute(SetReadMarkersTask.Params(roomId, markAllAsRead = true))
+            readMarkersTask.execute(SetReadMarkersTask.Params(roomId, forceReadMarker = true, forceReadReceipt = true))
         }
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/read/SetReadMarkersTask.kt

@@ -35,16 +35,16 @@ import io.realm.Realm
 import org.greenrobot.eventbus.EventBus
 import timber.log.Timber
 import javax.inject.Inject
-import kotlin.collections.HashMap
 import kotlin.collections.set
 
 internal interface SetReadMarkersTask : Task<SetReadMarkersTask.Params, Unit> {
 
     data class Params(
             val roomId: String,
-            val markAllAsRead: Boolean = false,
             val fullyReadEventId: String? = null,
-            val readReceiptEventId: String? = null
+            val readReceiptEventId: String? = null,
+            val forceReadReceipt: Boolean = false,
+            val forceReadMarker: Boolean = false
     )
 }
 
@@ -62,17 +62,18 @@ internal class DefaultSetReadMarkersTask @Inject constructor(
 
     override suspend fun execute(params: SetReadMarkersTask.Params) {
         val markers = HashMap<String, String>()
-
         Timber.v("Execute set read marker with params: $params")
-        val (fullyReadEventId, readReceiptEventId) = if (params.markAllAsRead) {
-            val latestSyncedEventId = Realm.getInstance(monarchy.realmConfiguration).use { realm ->
-                TimelineEventEntity.latestEvent(realm, roomId = params.roomId, includesSending = false)?.eventId
-            }
-            Pair(latestSyncedEventId, latestSyncedEventId)
+        val latestSyncedEventId = latestSyncedEventId(params.roomId)
+        val fullyReadEventId = if (params.forceReadMarker) {
+            latestSyncedEventId
         } else {
-            Pair(params.fullyReadEventId, params.readReceiptEventId)
+            params.fullyReadEventId
+        }
+        val readReceiptEventId = if (params.forceReadReceipt) {
+            latestSyncedEventId
+        } else {
+            params.readReceiptEventId
         }
-
         if (fullyReadEventId != null && !isReadMarkerMoreRecent(monarchy, params.roomId, fullyReadEventId)) {
             if (LocalEcho.isLocalEchoId(fullyReadEventId)) {
                 Timber.w("Can't set read marker for local event $fullyReadEventId")
@@ -80,7 +81,6 @@ internal class DefaultSetReadMarkersTask @Inject constructor(
                 markers[READ_MARKER] = fullyReadEventId
             }
         }
-
         if (readReceiptEventId != null
                 && !isEventRead(monarchy, userId, params.roomId, readReceiptEventId)) {
             if (LocalEcho.isLocalEchoId(readReceiptEventId)) {
@@ -89,16 +89,25 @@ internal class DefaultSetReadMarkersTask @Inject constructor(
                 markers[READ_RECEIPT] = readReceiptEventId
             }
         }
-        if (markers.isEmpty()) {
-            return
+
+        val shouldUpdateRoomSummary = readReceiptEventId != null && readReceiptEventId == latestSyncedEventId
+        if (markers.isNotEmpty() || shouldUpdateRoomSummary) {
+            updateDatabase(params.roomId, markers, shouldUpdateRoomSummary)
         }
-        updateDatabase(params.roomId, markers)
-        executeRequest<Unit>(eventBus) {
-            apiCall = roomAPI.sendReadMarker(params.roomId, markers)
+        if (markers.isNotEmpty()) {
+            executeRequest<Unit>(eventBus) {
+                isRetryable = true
+                apiCall = roomAPI.sendReadMarker(params.roomId, markers)
+            }
         }
     }
 
-    private suspend fun updateDatabase(roomId: String, markers: HashMap<String, String>) {
+    private fun latestSyncedEventId(roomId: String): String? =
+            Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+                TimelineEventEntity.latestEvent(realm, roomId = roomId, includesSending = false)?.eventId
+            }
+
+    private suspend fun updateDatabase(roomId: String, markers: HashMap<String, String>, shouldUpdateRoomSummary: Boolean) {
         monarchy.awaitTransaction { realm ->
             val readMarkerId = markers[READ_MARKER]
             val readReceiptId = markers[READ_RECEIPT]
@@ -108,14 +117,13 @@ internal class DefaultSetReadMarkersTask @Inject constructor(
             if (readReceiptId != null) {
                 val readReceiptContent = ReadReceiptHandler.createContent(userId, readReceiptId)
                 readReceiptHandler.handle(realm, roomId, readReceiptContent, false)
-                val isLatestReceived = TimelineEventEntity.latestEvent(realm, roomId = roomId, includesSending = false)?.eventId == readReceiptId
-                if (isLatestReceived) {
-                    val roomSummary = RoomSummaryEntity.where(realm, roomId).findFirst()
-                            ?: return@awaitTransaction
-                    roomSummary.notificationCount = 0
-                    roomSummary.highlightCount = 0
-                    roomSummary.hasUnreadMessages = false
-                }
+            }
+            if (shouldUpdateRoomSummary) {
+                val roomSummary = RoomSummaryEntity.where(realm, roomId).findFirst()
+                        ?: return@awaitTransaction
+                roomSummary.notificationCount = 0
+                roomSummary.highlightCount = 0
+                roomSummary.hasUnreadMessages = false
             }
         }
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/relation/DefaultRelationService.kt

@@ -233,6 +233,6 @@ internal class DefaultRelationService @AssistedInject constructor(
      * the same transaction id is received (in unsigned data)
      */
     private fun saveLocalEcho(event: Event) {
-        eventFactory.saveLocalEcho(monarchy, event)
+        eventFactory.createLocalEcho(event)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt

@@ -25,28 +25,24 @@ import com.squareup.inject.assisted.AssistedInject
 import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.api.session.content.ContentAttachmentData
 import im.vector.matrix.android.api.session.crypto.CryptoService
-import im.vector.matrix.android.api.session.events.model.*
-import im.vector.matrix.android.api.session.room.model.message.MessageContent
-import im.vector.matrix.android.api.session.room.model.message.MessageType
+import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.api.session.events.model.isImageMessage
+import im.vector.matrix.android.api.session.events.model.isTextMessage
 import im.vector.matrix.android.api.session.room.send.SendService
 import im.vector.matrix.android.api.session.room.send.SendState
 import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
 import im.vector.matrix.android.api.util.Cancelable
 import im.vector.matrix.android.api.util.CancelableBag
-import im.vector.matrix.android.internal.database.mapper.asDomain
-import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.model.RoomEntity
-import im.vector.matrix.android.internal.database.model.TimelineEventEntity
-import im.vector.matrix.android.internal.database.query.findAllInRoomWithSendStates
-import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionId
 import im.vector.matrix.android.internal.di.WorkManagerProvider
 import im.vector.matrix.android.internal.session.content.UploadContentWorker
 import im.vector.matrix.android.internal.session.room.timeline.TimelineSendEventWorkCommon
+import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.util.CancelableWork
 import im.vector.matrix.android.internal.worker.AlwaysSuccessfulWorker
 import im.vector.matrix.android.internal.worker.WorkerParamsFactory
 import im.vector.matrix.android.internal.worker.startChain
+import kotlinx.coroutines.launch
 import timber.log.Timber
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
@@ -61,7 +57,9 @@ internal class DefaultSendService @AssistedInject constructor(
         @SessionId private val sessionId: String,
         private val localEchoEventFactory: LocalEchoEventFactory,
         private val cryptoService: CryptoService,
-        private val monarchy: Monarchy
+        private val monarchy: Monarchy,
+        private val taskExecutor: TaskExecutor,
+        private val localEchoRepository: LocalEchoRepository
 ) : SendService {
 
     @AssistedInject.Factory
@@ -73,15 +71,14 @@ internal class DefaultSendService @AssistedInject constructor(
 
     override fun sendTextMessage(text: CharSequence, msgType: String, autoMarkdown: Boolean): Cancelable {
         val event = localEchoEventFactory.createTextEvent(roomId, msgType, text, autoMarkdown).also {
-            saveLocalEcho(it)
+            createLocalEcho(it)
         }
-
         return sendEvent(event)
     }
 
     override fun sendFormattedTextMessage(text: String, formattedText: String, msgType: String): Cancelable {
         val event = localEchoEventFactory.createFormattedTextEvent(roomId, TextContent(text, formattedText), msgType).also {
-            saveLocalEcho(it)
+            createLocalEcho(it)
         }
 
         return sendEvent(event)
@@ -156,174 +153,128 @@ internal class DefaultSendService @AssistedInject constructor(
     }
 
     override fun deleteFailedEcho(localEcho: TimelineEvent) {
-        monarchy.writeAsync { realm ->
-            TimelineEventEntity.where(realm, roomId = roomId, eventId = localEcho.root.eventId ?: "")
-                    .findFirst()
-                    ?.let { it.deleteFromRealm() }
-            EventEntity.where(realm, eventId = localEcho.root.eventId ?: "")
-                    .findFirst()
-                    ?.let { it.deleteFromRealm() }
+        taskExecutor.executorScope.launch {
+            localEchoRepository.deleteFailedEcho(roomId, localEcho)
         }
     }
 
-    override fun clearSendingQueue() {
-        timelineSendEventWorkCommon.cancelAllWorks(roomId)
-        workManagerProvider.workManager.cancelUniqueWork(buildWorkName(UPLOAD_WORK))
+        override fun clearSendingQueue() {
+            timelineSendEventWorkCommon.cancelAllWorks(roomId)
+            workManagerProvider.workManager.cancelUniqueWork(buildWorkName(UPLOAD_WORK))
 
-        // Replace the worker chains with a AlwaysSuccessfulWorker, to ensure the queues are well emptied
-        workManagerProvider.matrixOneTimeWorkRequestBuilder<AlwaysSuccessfulWorker>()
-                .build().let {
-                    timelineSendEventWorkCommon.postWork(roomId, it, ExistingWorkPolicy.REPLACE)
+            // Replace the worker chains with a AlwaysSuccessfulWorker, to ensure the queues are well emptied
+            workManagerProvider.matrixOneTimeWorkRequestBuilder<AlwaysSuccessfulWorker>()
+                    .build().let {
+                        timelineSendEventWorkCommon.postWork(roomId, it, ExistingWorkPolicy.REPLACE)
 
-                    // need to clear also image sending queue
-                    workManagerProvider.workManager
-                            .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.REPLACE, it)
-                            .enqueue()
-                }
-
-        monarchy.writeAsync { realm ->
-            RoomEntity.where(realm, roomId).findFirst()?.let { room ->
-                room.sendingTimelineEvents.forEach {
-                    it.root?.sendState = SendState.UNDELIVERED
-                }
+                        // need to clear also image sending queue
+                        workManagerProvider.workManager
+                                .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.REPLACE, it)
+                                .enqueue()
+                    }
+            taskExecutor.executorScope.launch {
+                localEchoRepository.clearSendingQueue(roomId)
             }
         }
-    }
 
-    override fun resendAllFailedMessages() {
-        monarchy.writeAsync { realm ->
-            TimelineEventEntity
-                    .findAllInRoomWithSendStates(realm, roomId, SendState.HAS_FAILED_STATES)
-                    .sortedBy { it.root?.originServerTs ?: 0 }
-                    .forEach { timelineEventEntity ->
-                        timelineEventEntity.root?.let {
-                            val event = it.asDomain()
-                            when (event.getClearType()) {
-                                EventType.MESSAGE,
-                                EventType.REDACTION,
-                                EventType.REACTION -> {
-                                    val content = event.getClearContent().toModel<MessageContent>()
-                                    if (content != null) {
-                                        when (content.type) {
-                                            MessageType.MSGTYPE_EMOTE,
-                                            MessageType.MSGTYPE_NOTICE,
-                                            MessageType.MSGTYPE_LOCATION,
-                                            MessageType.MSGTYPE_TEXT  -> {
-                                                it.sendState = SendState.UNSENT
-                                                sendEvent(event)
-                                            }
-                                            MessageType.MSGTYPE_FILE,
-                                            MessageType.MSGTYPE_VIDEO,
-                                            MessageType.MSGTYPE_IMAGE,
-                                            MessageType.MSGTYPE_AUDIO -> {
-                                                // need to resend the attachement
-                                            }
-                                            else                      -> {
-                                                Timber.e("Cannot resend message ${event.type} / ${content.type}")
-                                            }
-                                        }
-                                    } else {
-                                        Timber.e("Unsupported message to resend ${event.type}")
-                                    }
-                                }
-                                else               -> {
-                                    Timber.e("Unsupported message to resend ${event.type}")
-                                }
-                            }
-                        }
-                    }
-        }
-    }
-
-    override fun sendMedia(attachment: ContentAttachmentData): Cancelable {
-        // Create an event with the media file path
-        val event = localEchoEventFactory.createMediaEvent(roomId, attachment).also {
-            saveLocalEcho(it)
-        }
-
-        return internalSendMedia(event, attachment)
-    }
-
-    private fun internalSendMedia(localEcho: Event, attachment: ContentAttachmentData): Cancelable {
-        val isRoomEncrypted = cryptoService.isRoomEncrypted(roomId)
-
-        val uploadWork = createUploadMediaWork(localEcho, attachment, isRoomEncrypted, startChain = true)
-        val sendWork = createSendEventWork(localEcho, false)
-
-        if (isRoomEncrypted) {
-            val encryptWork = createEncryptEventWork(localEcho, false /*not start of chain, take input error*/)
-
-            val op: Operation = workManagerProvider.workManager
-                    .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND, uploadWork)
-                    .then(encryptWork)
-                    .then(sendWork)
-                    .enqueue()
-            op.result.addListener(Runnable {
-                if (op.result.isCancelled) {
-                    Timber.e("CHAIN WAS CANCELLED")
-                } else if (op.state.value is Operation.State.FAILURE) {
-                    Timber.e("CHAIN DID FAIL")
+        override fun resendAllFailedMessages() {
+            taskExecutor.executorScope.launch {
+                val eventsToResend = localEchoRepository.getAllFailedEventsToResend(roomId)
+                eventsToResend.forEach {
+                    sendEvent(it)
                 }
-            }, workerFutureListenerExecutor)
-        } else {
-            workManagerProvider.workManager
-                    .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND, uploadWork)
-                    .then(sendWork)
-                    .enqueue()
+                localEchoRepository.updateSendState(roomId, eventsToResend.mapNotNull { it.eventId }, SendState.UNSENT)
+            }
         }
 
-        return CancelableWork(workManagerProvider.workManager, sendWork.id)
-    }
-
-    private fun saveLocalEcho(event: Event) {
-        localEchoEventFactory.saveLocalEcho(monarchy, event)
-    }
-
-    private fun buildWorkName(identifier: String): String {
-        return "${roomId}_$identifier"
-    }
-
-    private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        // Same parameter
-        val params = EncryptEventWorker.Params(sessionId, roomId, event)
-        val sendWorkData = WorkerParamsFactory.toData(params)
-
-        return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
-                .setConstraints(WorkManagerProvider.workConstraints)
-                .setInputData(sendWorkData)
-                .startChain(startChain)
-                .setBackoffCriteria(BackoffPolicy.LINEAR, BACKOFF_DELAY, TimeUnit.MILLISECONDS)
-                .build()
-    }
-
-    private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId, roomId, event)
-        val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
-
-        return timelineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)
-    }
-
-    private fun createRedactEventWork(event: Event, reason: String?): OneTimeWorkRequest {
-        val redactEvent = localEchoEventFactory.createRedactEvent(roomId, event.eventId!!, reason).also {
-            saveLocalEcho(it)
+        override fun sendMedia(attachment: ContentAttachmentData): Cancelable {
+            // Create an event with the media file path
+            val event = localEchoEventFactory.createMediaEvent(roomId, attachment).also {
+                createLocalEcho(it)
+            }
+            return internalSendMedia(event, attachment)
         }
-        val sendContentWorkerParams = RedactEventWorker.Params(sessionId, redactEvent.eventId!!, roomId, event.eventId, reason)
-        val redactWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
-        return timelineSendEventWorkCommon.createWork<RedactEventWorker>(redactWorkData, true)
-    }
 
-    private fun createUploadMediaWork(event: Event,
-                                      attachment: ContentAttachmentData,
-                                      isRoomEncrypted: Boolean,
-                                      startChain: Boolean): OneTimeWorkRequest {
-        val uploadMediaWorkerParams = UploadContentWorker.Params(sessionId, roomId, event, attachment, isRoomEncrypted)
-        val uploadWorkData = WorkerParamsFactory.toData(uploadMediaWorkerParams)
+        private fun internalSendMedia(localEcho: Event, attachment: ContentAttachmentData): Cancelable {
+            val isRoomEncrypted = cryptoService.isRoomEncrypted(roomId)
 
-        return workManagerProvider.matrixOneTimeWorkRequestBuilder<UploadContentWorker>()
-                .setConstraints(WorkManagerProvider.workConstraints)
-                .startChain(startChain)
-                .setInputData(uploadWorkData)
-                .setBackoffCriteria(BackoffPolicy.LINEAR, BACKOFF_DELAY, TimeUnit.MILLISECONDS)
-                .build()
+            val uploadWork = createUploadMediaWork(localEcho, attachment, isRoomEncrypted, startChain = true)
+            val sendWork = createSendEventWork(localEcho, false)
+
+            if (isRoomEncrypted) {
+                val encryptWork = createEncryptEventWork(localEcho, false /*not start of chain, take input error*/)
+
+                val op: Operation = workManagerProvider.workManager
+                        .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND, uploadWork)
+                        .then(encryptWork)
+                        .then(sendWork)
+                        .enqueue()
+                op.result.addListener(Runnable {
+                    if (op.result.isCancelled) {
+                        Timber.e("CHAIN WAS CANCELLED")
+                    } else if (op.state.value is Operation.State.FAILURE) {
+                        Timber.e("CHAIN DID FAIL")
+                    }
+                }, workerFutureListenerExecutor)
+            } else {
+                workManagerProvider.workManager
+                        .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND, uploadWork)
+                        .then(sendWork)
+                        .enqueue()
+            }
+
+            return CancelableWork(workManagerProvider.workManager, sendWork.id)
+        }
+
+        private fun createLocalEcho(event: Event) {
+            localEchoEventFactory.createLocalEcho(event)
+        }
+
+        private fun buildWorkName(identifier: String): String {
+            return "${roomId}_$identifier"
+        }
+
+        private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
+            // Same parameter
+            val params = EncryptEventWorker.Params(sessionId, roomId, event)
+            val sendWorkData = WorkerParamsFactory.toData(params)
+
+            return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
+                    .setConstraints(WorkManagerProvider.workConstraints)
+                    .setInputData(sendWorkData)
+                    .startChain(startChain)
+                    .setBackoffCriteria(BackoffPolicy.LINEAR, BACKOFF_DELAY, TimeUnit.MILLISECONDS)
+                    .build()
+        }
+
+        private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
+            val sendContentWorkerParams = SendEventWorker.Params(sessionId, roomId, event)
+            val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
+
+            return timelineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)
+        }
+
+        private fun createRedactEventWork(event: Event, reason: String?): OneTimeWorkRequest {
+            val redactEvent = localEchoEventFactory.createRedactEvent(roomId, event.eventId!!, reason).also {
+                createLocalEcho(it)
+            }
+            val sendContentWorkerParams = RedactEventWorker.Params(sessionId, redactEvent.eventId!!, roomId, event.eventId, reason)
+            val redactWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
+            return timelineSendEventWorkCommon.createWork<RedactEventWorker>(redactWorkData, true)
+        }
+
+        private fun createUploadMediaWork(event: Event,
+                                          attachment: ContentAttachmentData,
+                                          isRoomEncrypted: Boolean,
+                                          startChain: Boolean): OneTimeWorkRequest {
+            val uploadMediaWorkerParams = UploadContentWorker.Params(sessionId, roomId, event, attachment, isRoomEncrypted)
+            val uploadWorkData = WorkerParamsFactory.toData(uploadMediaWorkerParams)
+
+            return workManagerProvider.matrixOneTimeWorkRequestBuilder<UploadContentWorker>()
+                    .setConstraints(WorkManagerProvider.workConstraints)
+                    .startChain(startChain)
+                    .setInputData(uploadWorkData)
+                    .setBackoffCriteria(BackoffPolicy.LINEAR, BACKOFF_DELAY, TimeUnit.MILLISECONDS)
+                    .build()
+        }
     }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt

@@ -18,26 +18,43 @@ package im.vector.matrix.android.internal.session.room.send
 
 import android.media.MediaMetadataRetriever
 import androidx.exifinterface.media.ExifInterface
-import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.R
 import im.vector.matrix.android.api.permalinks.PermalinkFactory
 import im.vector.matrix.android.api.session.content.ContentAttachmentData
-import im.vector.matrix.android.api.session.events.model.*
-import im.vector.matrix.android.api.session.room.model.message.*
+import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.LocalEcho
+import im.vector.matrix.android.api.session.events.model.RelationType
+import im.vector.matrix.android.api.session.events.model.UnsignedData
+import im.vector.matrix.android.api.session.events.model.toContent
+import im.vector.matrix.android.api.session.events.model.toModel
+import im.vector.matrix.android.api.session.room.model.message.AudioInfo
+import im.vector.matrix.android.api.session.room.model.message.FileInfo
+import im.vector.matrix.android.api.session.room.model.message.ImageInfo
+import im.vector.matrix.android.api.session.room.model.message.MessageAudioContent
+import im.vector.matrix.android.api.session.room.model.message.MessageContent
+import im.vector.matrix.android.api.session.room.model.message.MessageFileContent
+import im.vector.matrix.android.api.session.room.model.message.MessageFormat
+import im.vector.matrix.android.api.session.room.model.message.MessageImageContent
+import im.vector.matrix.android.api.session.room.model.message.MessageTextContent
+import im.vector.matrix.android.api.session.room.model.message.MessageType
+import im.vector.matrix.android.api.session.room.model.message.MessageVerificationRequestContent
+import im.vector.matrix.android.api.session.room.model.message.MessageVideoContent
+import im.vector.matrix.android.api.session.room.model.message.ThumbnailInfo
+import im.vector.matrix.android.api.session.room.model.message.VideoInfo
+import im.vector.matrix.android.api.session.room.model.message.isReply
 import im.vector.matrix.android.api.session.room.model.relation.ReactionContent
 import im.vector.matrix.android.api.session.room.model.relation.ReactionInfo
 import im.vector.matrix.android.api.session.room.model.relation.RelationDefaultContent
 import im.vector.matrix.android.api.session.room.model.relation.ReplyToContent
 import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
 import im.vector.matrix.android.api.session.room.timeline.getLastMessageContent
-import im.vector.matrix.android.internal.database.helper.addSendingEvent
-import im.vector.matrix.android.internal.database.model.RoomEntity
-import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.UserId
 import im.vector.matrix.android.internal.session.content.ThumbnailExtractor
-import im.vector.matrix.android.internal.session.room.RoomSummaryUpdater
 import im.vector.matrix.android.internal.session.room.send.pills.TextPillsUtils
+import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.util.StringProvider
+import kotlinx.coroutines.launch
 import org.commonmark.parser.Parser
 import org.commonmark.renderer.html.HtmlRenderer
 import javax.inject.Inject
@@ -54,8 +71,9 @@ import javax.inject.Inject
 internal class LocalEchoEventFactory @Inject constructor(
         @UserId private val userId: String,
         private val stringProvider: StringProvider,
-        private val roomSummaryUpdater: RoomSummaryUpdater,
-        private val textPillsUtils: TextPillsUtils
+        private val textPillsUtils: TextPillsUtils,
+        private val taskExecutor: TaskExecutor,
+        private val localEchoRepository: LocalEchoRepository
 ) {
     // TODO Inject
     private val parser = Parser.builder().build()
@@ -66,7 +84,7 @@ internal class LocalEchoEventFactory @Inject constructor(
         if (msgType == MessageType.MSGTYPE_TEXT || msgType == MessageType.MSGTYPE_EMOTE) {
             return createFormattedTextEvent(roomId, createTextContent(text, autoMarkdown), msgType)
         }
-        val content = MessageTextContent(type = msgType, body = text.toString())
+        val content = MessageTextContent(msgType = msgType, body = text.toString())
         return createEvent(roomId, content)
     }
 
@@ -105,7 +123,7 @@ internal class LocalEchoEventFactory @Inject constructor(
                                compatibilityText: String): Event {
         return createEvent(roomId,
                 MessageTextContent(
-                        type = msgType,
+                        msgType = msgType,
                         body = compatibilityText,
                         relatesTo = RelationDefaultContent(RelationType.REPLACE, targetEventId),
                         newContent = createTextContent(newBodyText, newBodyAutoMarkdown)
@@ -114,7 +132,8 @@ internal class LocalEchoEventFactory @Inject constructor(
                 ))
     }
 
-    fun createReplaceTextOfReply(roomId: String, eventReplaced: TimelineEvent,
+    fun createReplaceTextOfReply(roomId: String,
+                                 eventReplaced: TimelineEvent,
                                  originalEvent: TimelineEvent,
                                  newBodyText: String,
                                  newBodyAutoMarkdown: Boolean,
@@ -140,12 +159,12 @@ internal class LocalEchoEventFactory @Inject constructor(
 
         return createEvent(roomId,
                 MessageTextContent(
-                        type = msgType,
+                        msgType = msgType,
                         body = compatibilityText,
                         relatesTo = RelationDefaultContent(RelationType.REPLACE, eventReplaced.root.eventId),
                         newContent = MessageTextContent(
-                                type = msgType,
-                                format = MessageType.FORMAT_MATRIX_HTML,
+                                msgType = msgType,
+                                format = MessageFormat.FORMAT_MATRIX_HTML,
                                 body = replyFallback,
                                 formattedBody = replyFormatted
                         )
@@ -197,7 +216,7 @@ internal class LocalEchoEventFactory @Inject constructor(
         }
 
         val content = MessageImageContent(
-                type = MessageType.MSGTYPE_IMAGE,
+                msgType = MessageType.MSGTYPE_IMAGE,
                 body = attachment.name ?: "image",
                 info = ImageInfo(
                         mimeType = attachment.mimeType,
@@ -229,7 +248,7 @@ internal class LocalEchoEventFactory @Inject constructor(
             )
         }
         val content = MessageVideoContent(
-                type = MessageType.MSGTYPE_VIDEO,
+                msgType = MessageType.MSGTYPE_VIDEO,
                 body = attachment.name ?: "video",
                 videoInfo = VideoInfo(
                         mimeType = attachment.mimeType,
@@ -248,7 +267,7 @@ internal class LocalEchoEventFactory @Inject constructor(
 
     private fun createAudioEvent(roomId: String, attachment: ContentAttachmentData): Event {
         val content = MessageAudioContent(
-                type = MessageType.MSGTYPE_AUDIO,
+                msgType = MessageType.MSGTYPE_AUDIO,
                 body = attachment.name ?: "audio",
                 audioInfo = AudioInfo(
                         mimeType = attachment.mimeType?.takeIf { it.isNotBlank() } ?: "audio/mpeg",
@@ -261,7 +280,7 @@ internal class LocalEchoEventFactory @Inject constructor(
 
     private fun createFileEvent(roomId: String, attachment: ContentAttachmentData): Event {
         val content = MessageFileContent(
-                type = MessageType.MSGTYPE_FILE,
+                msgType = MessageType.MSGTYPE_FILE,
                 body = attachment.name ?: "file",
                 info = FileInfo(
                         mimeType = attachment.mimeType?.takeIf { it.isNotBlank() }
@@ -332,8 +351,8 @@ internal class LocalEchoEventFactory @Inject constructor(
 
         val eventId = eventReplied.root.eventId ?: return null
         val content = MessageTextContent(
-                type = MessageType.MSGTYPE_TEXT,
-                format = MessageType.FORMAT_MATRIX_HTML,
+                msgType = MessageType.MSGTYPE_TEXT,
+                format = MessageFormat.FORMAT_MATRIX_HTML,
                 body = replyFallback,
                 formattedBody = replyFormatted,
                 relatesTo = RelationDefaultContent(null, null, ReplyToContent(eventId))
@@ -366,13 +385,13 @@ internal class LocalEchoEventFactory @Inject constructor(
      * himself a reply, but it will contain the fallbacks, so we have to trim them.
      */
     private fun bodyForReply(content: MessageContent?, originalContent: MessageContent?): TextContent {
-        when (content?.type) {
+        when (content?.msgType) {
             MessageType.MSGTYPE_EMOTE,
             MessageType.MSGTYPE_TEXT,
             MessageType.MSGTYPE_NOTICE -> {
                 var formattedText: String? = null
                 if (content is MessageTextContent) {
-                    if (content.format == MessageType.FORMAT_MATRIX_HTML) {
+                    if (content.format == MessageFormat.FORMAT_MATRIX_HTML) {
                         formattedText = content.formattedBody
                     }
                 }
@@ -421,13 +440,10 @@ internal class LocalEchoEventFactory @Inject constructor(
         )
     }
 
-    fun saveLocalEcho(monarchy: Monarchy, event: Event) {
+    fun createLocalEcho(event: Event) {
         checkNotNull(event.roomId) { "Your event should have a roomId" }
-        monarchy.writeAsync { realm ->
-            val roomEntity = RoomEntity.where(realm, roomId = event.roomId).findFirst()
-                    ?: return@writeAsync
-            roomEntity.addSendingEvent(event)
-            roomSummaryUpdater.update(realm, event.roomId)
+        taskExecutor.executorScope.launch {
+            localEchoRepository.createLocalEcho(event)
         }
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoRepository.kt

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.session.room.send
+
+import com.zhuinden.monarchy.Monarchy
+import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.toModel
+import im.vector.matrix.android.api.session.room.model.message.MessageContent
+import im.vector.matrix.android.api.session.room.model.message.MessageType
+import im.vector.matrix.android.api.session.room.send.SendState
+import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
+import im.vector.matrix.android.internal.database.helper.nextId
+import im.vector.matrix.android.internal.database.mapper.TimelineEventMapper
+import im.vector.matrix.android.internal.database.mapper.asDomain
+import im.vector.matrix.android.internal.database.mapper.toEntity
+import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
+import im.vector.matrix.android.internal.database.query.findAllInRoomWithSendStates
+import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.session.room.RoomSummaryUpdater
+import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
+import im.vector.matrix.android.internal.session.room.timeline.DefaultTimeline
+import im.vector.matrix.android.internal.util.awaitTransaction
+import io.realm.Realm
+import org.greenrobot.eventbus.EventBus
+import timber.log.Timber
+import java.lang.IllegalStateException
+import javax.inject.Inject
+
+internal class LocalEchoRepository @Inject constructor(private val monarchy: Monarchy,
+                                                       private val roomSummaryUpdater: RoomSummaryUpdater,
+                                                       private val eventBus: EventBus,
+                                                       private val timelineEventMapper: TimelineEventMapper) {
+
+    suspend fun createLocalEcho(event: Event) {
+        val roomId = event.roomId ?: throw IllegalStateException("You should have set a roomId for your event")
+        val senderId = event.senderId ?: throw IllegalStateException("You should have set a senderIf for your event")
+        if (event.eventId == null) {
+            throw IllegalStateException("You should have set an eventId for your event")
+        }
+        val timelineEventEntity = Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+            val eventEntity = event.toEntity(roomId, SendState.UNSENT)
+            val roomMemberHelper = RoomMemberHelper(realm, roomId)
+            val myUser = roomMemberHelper.getLastRoomMember(senderId)
+            val localId = TimelineEventEntity.nextId(realm)
+            TimelineEventEntity(localId).also {
+                it.root = eventEntity
+                it.eventId = event.eventId
+                it.roomId = roomId
+                it.senderName = myUser?.displayName
+                it.senderAvatar = myUser?.avatarUrl
+                it.isUniqueDisplayName = roomMemberHelper.isUniqueDisplayName(myUser?.displayName)
+            }
+        }
+        val timelineEvent = timelineEventMapper.map(timelineEventEntity)
+        eventBus.post(DefaultTimeline.OnLocalEchoCreated(roomId = roomId, timelineEvent = timelineEvent))
+        monarchy.awaitTransaction { realm ->
+            val roomEntity = RoomEntity.where(realm, roomId = roomId).findFirst() ?: return@awaitTransaction
+            roomEntity.sendingTimelineEvents.add(0, timelineEventEntity)
+            roomSummaryUpdater.update(realm, roomId)
+        }
+    }
+
+    suspend fun deleteFailedEcho(roomId: String, localEcho: TimelineEvent) {
+        monarchy.awaitTransaction { realm ->
+            TimelineEventEntity.where(realm, roomId = roomId, eventId = localEcho.root.eventId ?: "").findFirst()?.deleteFromRealm()
+            EventEntity.where(realm, eventId = localEcho.root.eventId ?: "").findFirst()?.deleteFromRealm()
+        }
+    }
+
+    suspend fun clearSendingQueue(roomId: String) {
+        monarchy.awaitTransaction { realm ->
+            RoomEntity.where(realm, roomId).findFirst()?.let { room ->
+                room.sendingTimelineEvents.forEach {
+                    it.root?.sendState = SendState.UNDELIVERED
+                }
+            }
+        }
+    }
+
+    suspend fun updateSendState(roomId: String, eventIds: List<String>, sendState: SendState) {
+        monarchy.awaitTransaction { realm ->
+            val timelineEvents = TimelineEventEntity.where(realm, roomId, eventIds).findAll()
+            timelineEvents.forEach {
+                it.root?.sendState = sendState
+            }
+        }
+    }
+
+    fun getAllFailedEventsToResend(roomId: String): List<Event> {
+        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+            TimelineEventEntity
+                    .findAllInRoomWithSendStates(realm, roomId, SendState.HAS_FAILED_STATES)
+                    .sortedByDescending { it.displayIndex }
+                    .mapNotNull { it.root?.asDomain() }
+                    .filter { event ->
+                        when (event.getClearType()) {
+                            EventType.MESSAGE,
+                            EventType.REDACTION,
+                            EventType.REACTION -> {
+                                val content = event.getClearContent().toModel<MessageContent>()
+                                if (content != null) {
+                                    when (content.msgType) {
+                                        MessageType.MSGTYPE_EMOTE,
+                                        MessageType.MSGTYPE_NOTICE,
+                                        MessageType.MSGTYPE_LOCATION,
+                                        MessageType.MSGTYPE_TEXT  -> {
+                                            true
+                                        }
+                                        MessageType.MSGTYPE_FILE,
+                                        MessageType.MSGTYPE_VIDEO,
+                                        MessageType.MSGTYPE_IMAGE,
+                                        MessageType.MSGTYPE_AUDIO -> {
+                                            // need to resend the attachment
+                                            false
+                                        }
+                                        else                      -> {
+                                            Timber.e("Cannot resend message ${event.type} / ${content.msgType}")
+                                            false
+                                        }
+                                    }
+                                } else {
+                                    Timber.e("Unsupported message to resend ${event.type}")
+                                    false
+                                }
+                            }
+                            else               -> {
+                                Timber.e("Unsupported message to resend ${event.type}")
+                                false
+                            }
+                        }
+                    }
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/TextContent.kt

@@ -16,6 +16,7 @@
 
 package im.vector.matrix.android.internal.session.room.send
 
+import im.vector.matrix.android.api.session.room.model.message.MessageFormat
 import im.vector.matrix.android.api.session.room.model.message.MessageTextContent
 import im.vector.matrix.android.api.session.room.model.message.MessageType
 import im.vector.matrix.android.api.util.ContentUtils.extractUsefulTextFromHtmlReply
@@ -33,8 +34,8 @@ data class TextContent(
 
 fun TextContent.toMessageTextContent(msgType: String = MessageType.MSGTYPE_TEXT): MessageTextContent {
     return MessageTextContent(
-            type = msgType,
-            format = MessageType.FORMAT_MATRIX_HTML.takeIf { formattedText != null },
+            msgType = msgType,
+            format = MessageFormat.FORMAT_MATRIX_HTML.takeIf { formattedText != null },
             body = text,
             formattedBody = formattedText
     )

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/state/DefaultStateService.kt

@@ -28,10 +28,9 @@ import im.vector.matrix.android.api.session.room.state.StateService
 import im.vector.matrix.android.api.util.Optional
 import im.vector.matrix.android.api.util.toOptional
 import im.vector.matrix.android.internal.database.mapper.asDomain
-import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.descending
-import im.vector.matrix.android.internal.database.query.prev
-import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
+import im.vector.matrix.android.internal.database.query.getOrNull
+import im.vector.matrix.android.internal.database.query.whereStateKey
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.configureWith
 import io.realm.Realm
@@ -47,16 +46,16 @@ internal class DefaultStateService @AssistedInject constructor(@Assisted private
         fun create(roomId: String): StateService
     }
 
-    override fun getStateEvent(eventType: String): Event? {
+    override fun getStateEvent(eventType: String, stateKey: String): Event? {
         return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
-            EventEntity.where(realm, roomId, eventType).prev()?.asDomain()
+            CurrentStateEventEntity.getOrNull(realm, roomId, type = eventType, stateKey = stateKey)?.root?.asDomain()
         }
     }
 
-    override fun getStateEventLive(eventType: String): LiveData<Optional<Event>> {
+    override fun getStateEventLive(eventType: String, stateKey: String): LiveData<Optional<Event>> {
         val liveData = monarchy.findAllMappedWithChanges(
-                { realm -> EventEntity.where(realm, roomId, eventType).descending() },
-                { it.asDomain() }
+                { realm -> CurrentStateEventEntity.whereStateKey(realm, roomId, type = eventType, stateKey = "") },
+                { it.root?.asDomain() }
         )
         return Transformations.map(liveData) { results ->
             results.firstOrNull().toOptional()

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/ClearUnlinkedEventsTask.kt

@@ -1,47 +0,0 @@
-/*
-
-  * Copyright 2019 New Vector Ltd
-  *
-  * Licensed under the Apache License, Version 2.0 (the "License");
-  * you may not use this file except in compliance with the License.
-  * You may obtain a copy of the License at
-  *
-  *     http://www.apache.org/licenses/LICENSE-2.0
-  *
-  * Unless required by applicable law or agreed to in writing, software
-  * distributed under the License is distributed on an "AS IS" BASIS,
-  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  * See the License for the specific language governing permissions and
-  * limitations under the License.
-
- */
-package im.vector.matrix.android.internal.session.room.timeline
-
-import com.zhuinden.monarchy.Monarchy
-import im.vector.matrix.android.internal.database.helper.deleteOnCascade
-import im.vector.matrix.android.internal.database.model.ChunkEntity
-import im.vector.matrix.android.internal.database.model.ChunkEntityFields
-import im.vector.matrix.android.internal.database.query.where
-import im.vector.matrix.android.internal.task.Task
-import im.vector.matrix.android.internal.util.awaitTransaction
-import javax.inject.Inject
-
-internal interface ClearUnlinkedEventsTask : Task<ClearUnlinkedEventsTask.Params, Unit> {
-
-    data class Params(val roomId: String)
-}
-
-internal class DefaultClearUnlinkedEventsTask @Inject constructor(private val monarchy: Monarchy) : ClearUnlinkedEventsTask {
-
-    override suspend fun execute(params: ClearUnlinkedEventsTask.Params) {
-        monarchy.awaitTransaction { localRealm ->
-            val unlinkedChunks = ChunkEntity
-                    .where(localRealm, roomId = params.roomId)
-                    .equalTo(ChunkEntityFields.IS_UNLINKED, true)
-                    .findAll()
-            unlinkedChunks.forEach {
-                it.deleteOnCascade()
-            }
-        }
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultGetContextOfEventTask.kt

@@ -27,8 +27,7 @@ internal interface GetContextOfEventTask : Task<GetContextOfEventTask.Params, To
 
     data class Params(
             val roomId: String,
-            val eventId: String,
-            val limit: Int
+            val eventId: String
     )
 }
 
@@ -42,8 +41,9 @@ internal class DefaultGetContextOfEventTask @Inject constructor(
     override suspend fun execute(params: GetContextOfEventTask.Params): TokenChunkEventPersistor.Result {
         val filter = filterRepository.getRoomFilter()
         val response = executeRequest<EventContextResponse>(eventBus) {
-            apiCall = roomAPI.getContextOfEvent(params.roomId, params.eventId, params.limit, filter)
+            // We are limiting the response to the event with eventId to be sure we don't have any issue with potential merging process.
+            apiCall = roomAPI.getContextOfEvent(params.roomId, params.eventId, 0, filter)
         }
-        return tokenChunkEventPersistor.insertInDb(response, params.roomId, PaginationDirection.BACKWARDS)
+        return tokenChunkEventPersistor.insertInDb(response, params.roomId, PaginationDirection.FORWARDS)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultPaginationTask.kt

@@ -43,6 +43,7 @@ internal class DefaultPaginationTask @Inject constructor(
     override suspend fun execute(params: PaginationTask.Params): TokenChunkEventPersistor.Result {
         val filter = filterRepository.getRoomFilter()
         val chunk = executeRequest<PaginationResponse>(eventBus) {
+            isRetryable = true
             apiCall = roomAPI.getRoomMessagesFrom(params.roomId, params.from, params.direction.value, params.limit, filter)
         }
         return tokenChunkEventPersistor.insertInDb(chunk, params.roomId, params.direction)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultTimeline.kt

@@ -19,7 +19,10 @@ package im.vector.matrix.android.internal.session.room.timeline
 import im.vector.matrix.android.api.MatrixCallback
 import im.vector.matrix.android.api.session.crypto.CryptoService
 import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.RelationType
+import im.vector.matrix.android.api.session.events.model.toModel
 import im.vector.matrix.android.api.session.room.model.ReadReceipt
+import im.vector.matrix.android.api.session.room.model.message.MessageContent
 import im.vector.matrix.android.api.session.room.send.SendState
 import im.vector.matrix.android.api.session.room.timeline.Timeline
 import im.vector.matrix.android.api.session.room.timeline.TimelineEvent
@@ -27,27 +30,39 @@ import im.vector.matrix.android.api.session.room.timeline.TimelineSettings
 import im.vector.matrix.android.api.util.CancelableBag
 import im.vector.matrix.android.internal.database.mapper.TimelineEventMapper
 import im.vector.matrix.android.internal.database.mapper.asDomain
-import im.vector.matrix.android.internal.database.model.*
+import im.vector.matrix.android.internal.database.model.ChunkEntity
+import im.vector.matrix.android.internal.database.model.ChunkEntityFields
+import im.vector.matrix.android.internal.database.model.EventAnnotationsSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields
 import im.vector.matrix.android.internal.database.query.FilterContent
 import im.vector.matrix.android.internal.database.query.findAllInRoomWithSendStates
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.database.query.whereInRoom
-import im.vector.matrix.android.internal.task.TaskConstraints
+import im.vector.matrix.android.internal.database.query.whereRoomId
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.task.configureWith
 import im.vector.matrix.android.internal.util.Debouncer
 import im.vector.matrix.android.internal.util.createBackgroundHandler
 import im.vector.matrix.android.internal.util.createUIHandler
-import io.realm.*
+import io.realm.OrderedCollectionChangeSet
+import io.realm.OrderedRealmCollectionChangeListener
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.RealmQuery
+import io.realm.RealmResults
+import io.realm.Sort
+import org.greenrobot.eventbus.EventBus
+import org.greenrobot.eventbus.Subscribe
+import org.greenrobot.eventbus.ThreadMode
 import timber.log.Timber
-import java.util.*
+import java.util.Collections
+import java.util.UUID
 import java.util.concurrent.CopyOnWriteArrayList
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
-import kotlin.collections.ArrayList
-import kotlin.collections.HashMap
 import kotlin.math.max
-import kotlin.math.min
 
 private const val MIN_FETCHING_COUNT = 30
 
@@ -57,14 +72,17 @@ internal class DefaultTimeline(
         private val realmConfiguration: RealmConfiguration,
         private val taskExecutor: TaskExecutor,
         private val contextOfEventTask: GetContextOfEventTask,
-        private val clearUnlinkedEventsTask: ClearUnlinkedEventsTask,
         private val paginationTask: PaginationTask,
         private val cryptoService: CryptoService,
         private val timelineEventMapper: TimelineEventMapper,
         private val settings: TimelineSettings,
-        private val hiddenReadReceipts: TimelineHiddenReadReceipts
+        private val hiddenReadReceipts: TimelineHiddenReadReceipts,
+        private val eventBus: EventBus
 ) : Timeline, TimelineHiddenReadReceipts.Delegate {
 
+    data class OnNewTimelineEvents(val roomId: String, val eventIds: List<String>)
+    data class OnLocalEchoCreated(val roomId: String, val timelineEvent: TimelineEvent)
+
     companion object {
         val BACKGROUND_HANDLER = createBackgroundHandler("TIMELINE_DB_THREAD")
     }
@@ -85,6 +103,7 @@ internal class DefaultTimeline(
 
     private var prevDisplayIndex: Int? = null
     private var nextDisplayIndex: Int? = null
+    private val inMemorySendingEvents = Collections.synchronizedList<TimelineEvent>(ArrayList())
     private val builtEvents = Collections.synchronizedList<TimelineEvent>(ArrayList())
     private val builtEventsIdMap = Collections.synchronizedMap(HashMap<String, Int>())
     private val backwardsState = AtomicReference(State())
@@ -101,7 +120,7 @@ internal class DefaultTimeline(
         if (!results.isLoaded || !results.isValid) {
             return@OrderedRealmCollectionChangeListener
         }
-        handleUpdates(changeSet)
+        handleUpdates(results, changeSet)
     }
 
     private val relationsListener = OrderedRealmCollectionChangeListener<RealmResults<EventAnnotationsSummaryEntity>> { collection, changeSet ->
@@ -118,7 +137,7 @@ internal class DefaultTimeline(
         if (hasChange) postSnapshot()
     }
 
-// Public methods ******************************************************************************
+    // Public methods ******************************************************************************
 
     override fun paginate(direction: Timeline.Direction, count: Int) {
         BACKGROUND_HANDLER.post {
@@ -149,6 +168,7 @@ internal class DefaultTimeline(
     override fun start() {
         if (isStarted.compareAndSet(false, true)) {
             Timber.v("Start timeline for roomId: $roomId and eventId: $initialEventId")
+            eventBus.register(this)
             BACKGROUND_HANDLER.post {
                 eventDecryptor.start()
                 val realm = Realm.getInstance(realmConfiguration)
@@ -160,18 +180,18 @@ internal class DefaultTimeline(
                     }
                 }
 
-                nonFilteredEvents = buildEventQuery(realm).sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.DESCENDING).findAll()
+                nonFilteredEvents = buildEventQuery(realm).sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING).findAll()
                 filteredEvents = nonFilteredEvents.where()
                         .filterEventsWithSettings()
                         .findAll()
                 handleInitialLoad()
-                filteredEvents.addChangeListener(eventsChangeListener)
+                nonFilteredEvents.addChangeListener(eventsChangeListener)
 
                 eventRelations = EventAnnotationsSummaryEntity.whereInRoom(realm, roomId)
                         .findAllAsync()
                         .also { it.addChangeListener(relationsListener) }
 
-                if (settings.buildReadReceipts) {
+                if (settings.shouldHandleHiddenReadReceipts()) {
                     hiddenReadReceipts.start(realm, filteredEvents, nonFilteredEvents, this)
                 }
                 isReady.set(true)
@@ -179,9 +199,14 @@ internal class DefaultTimeline(
         }
     }
 
+    private fun TimelineSettings.shouldHandleHiddenReadReceipts(): Boolean {
+        return buildReadReceipts && (filterEdits || filterTypes)
+    }
+
     override fun dispose() {
         if (isStarted.compareAndSet(true, false)) {
             isReady.set(false)
+            eventBus.unregister(this)
             Timber.v("Dispose timeline for roomId: $roomId and eventId: $initialEventId")
             cancelableBag.cancel()
             BACKGROUND_HANDLER.removeCallbacksAndMessages(null)
@@ -190,10 +215,10 @@ internal class DefaultTimeline(
                 if (this::eventRelations.isInitialized) {
                     eventRelations.removeAllChangeListeners()
                 }
-                if (this::filteredEvents.isInitialized) {
-                    filteredEvents.removeAllChangeListeners()
+                if (this::nonFilteredEvents.isInitialized) {
+                    nonFilteredEvents.removeAllChangeListeners()
                 }
-                if (settings.buildReadReceipts) {
+                if (settings.shouldHandleHiddenReadReceipts()) {
                     hiddenReadReceipts.dispose()
                 }
                 clearAllValues()
@@ -202,9 +227,6 @@ internal class DefaultTimeline(
                 }
                 eventDecryptor.destroy()
             }
-            clearUnlinkedEventsTask
-                    .configureWith(ClearUnlinkedEventsTask.Params(roomId))
-                    .executeBy(taskExecutor)
         }
     }
 
@@ -238,7 +260,7 @@ internal class DefaultTimeline(
         // Otherwise, we should check if the event is in the db, but is hidden because of filters
         return Realm.getInstance(realmConfiguration).use { localRealm ->
             val nonFilteredEvents = buildEventQuery(localRealm)
-                    .sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.DESCENDING)
+                    .sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING)
                     .findAll()
 
             val nonFilteredEvent = nonFilteredEvents.where()
@@ -253,11 +275,11 @@ internal class DefaultTimeline(
                     .findFirst() == null
 
             if (isHidden) {
-                val displayIndex = nonFilteredEvent?.root?.displayIndex
+                val displayIndex = nonFilteredEvent?.displayIndex
                 if (displayIndex != null) {
                     // Then we are looking for the first displayable event after the hidden one
                     val firstDisplayedEvent = filteredEvents.where()
-                            .lessThanOrEqualTo(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, displayIndex)
+                            .lessThanOrEqualTo(TimelineEventEntityFields.DISPLAY_INDEX, displayIndex)
                             .findFirst()
                     firstDisplayedEvent?.eventId
                 } else {
@@ -302,6 +324,26 @@ internal class DefaultTimeline(
         postSnapshot()
     }
 
+    @Subscribe(threadMode = ThreadMode.MAIN)
+    fun onNewTimelineEvents(onNewTimelineEvents: OnNewTimelineEvents) {
+        if (isLive && onNewTimelineEvents.roomId == roomId) {
+            listeners.forEach {
+                it.onNewTimelineEvents(onNewTimelineEvents.eventIds)
+            }
+        }
+    }
+
+    @Subscribe(threadMode = ThreadMode.MAIN)
+    fun onLocalEchoCreated(onLocalEchoCreated: OnLocalEchoCreated) {
+        if (isLive && onLocalEchoCreated.roomId == roomId) {
+            listeners.forEach {
+                it.onNewTimelineEvents(listOf(onLocalEchoCreated.timelineEvent.eventId))
+            }
+            inMemorySendingEvents.add(0, onLocalEchoCreated.timelineEvent)
+            postSnapshot()
+        }
+    }
+
     // Private methods *****************************************************************************
 
     private fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent): Boolean {
@@ -327,14 +369,14 @@ internal class DefaultTimeline(
 
         updateState(Timeline.Direction.FORWARDS) {
             it.copy(
-                    hasMoreInCache = firstBuiltEvent == null || firstBuiltEvent.displayIndex < firstCacheEvent?.root?.displayIndex ?: Int.MIN_VALUE,
+                    hasMoreInCache = firstBuiltEvent == null || firstBuiltEvent.displayIndex < firstCacheEvent?.displayIndex ?: Int.MIN_VALUE,
                     hasReachedEnd = chunkEntity?.isLastForward ?: false
             )
         }
 
         updateState(Timeline.Direction.BACKWARDS) {
             it.copy(
-                    hasMoreInCache = lastBuiltEvent == null || lastBuiltEvent.displayIndex > lastCacheEvent?.root?.displayIndex ?: Int.MAX_VALUE,
+                    hasMoreInCache = lastBuiltEvent == null || lastBuiltEvent.displayIndex > lastCacheEvent?.displayIndex ?: Int.MAX_VALUE,
                     hasReachedEnd = chunkEntity?.isLastBackward ?: false || lastCacheEvent?.root?.type == EventType.STATE_ROOM_CREATE
             )
         }
@@ -346,10 +388,9 @@ internal class DefaultTimeline(
      */
     private fun paginateInternal(startDisplayIndex: Int?,
                                  direction: Timeline.Direction,
-                                 count: Int,
-                                 strict: Boolean = false): Boolean {
+                                 count: Int): Boolean {
         updateState(direction) { it.copy(requestedPaginationCount = count, isPaginating = true) }
-        val builtCount = buildTimelineEvents(startDisplayIndex, direction, count.toLong(), strict)
+        val builtCount = buildTimelineEvents(startDisplayIndex, direction, count.toLong())
         val shouldFetchMore = builtCount < count && !hasReachedEnd(direction)
         if (shouldFetchMore) {
             val newRequestedCount = count - builtCount
@@ -359,7 +400,6 @@ internal class DefaultTimeline(
         } else {
             updateState(direction) { it.copy(isPaginating = false, requestedPaginationCount = 0) }
         }
-
         return !shouldFetchMore
     }
 
@@ -370,12 +410,15 @@ internal class DefaultTimeline(
     private fun buildSendingEvents(): List<TimelineEvent> {
         val sendingEvents = ArrayList<TimelineEvent>()
         if (hasReachedEnd(Timeline.Direction.FORWARDS) && !hasMoreInCache(Timeline.Direction.FORWARDS)) {
+            sendingEvents.addAll(inMemorySendingEvents.filterEventsWithSettings())
             roomEntity?.sendingTimelineEvents
                     ?.where()
                     ?.filterEventsWithSettings()
                     ?.findAll()
-                    ?.forEach {
-                        sendingEvents.add(timelineEventMapper.map(it))
+                    ?.forEach { timelineEventEntity ->
+                        if (sendingEvents.find { it.eventId == timelineEventEntity.eventId } == null) {
+                            sendingEvents.add(timelineEventMapper.map(timelineEventEntity))
+                        }
                     }
         }
         return sendingEvents
@@ -387,14 +430,14 @@ internal class DefaultTimeline(
 
     private fun getState(direction: Timeline.Direction): State {
         return when (direction) {
-            Timeline.Direction.FORWARDS -> forwardsState.get()
+            Timeline.Direction.FORWARDS  -> forwardsState.get()
             Timeline.Direction.BACKWARDS -> backwardsState.get()
         }
     }
 
     private fun updateState(direction: Timeline.Direction, update: (State) -> State) {
         val stateReference = when (direction) {
-            Timeline.Direction.FORWARDS -> forwardsState
+            Timeline.Direction.FORWARDS  -> forwardsState
             Timeline.Direction.BACKWARDS -> backwardsState
         }
         val currentValue = stateReference.get()
@@ -409,26 +452,26 @@ internal class DefaultTimeline(
         var shouldFetchInitialEvent = false
         val currentInitialEventId = initialEventId
         val initialDisplayIndex = if (currentInitialEventId == null) {
-            filteredEvents.firstOrNull()?.root?.displayIndex
+            nonFilteredEvents.firstOrNull()?.displayIndex
         } else {
             val initialEvent = nonFilteredEvents.where()
                     .equalTo(TimelineEventEntityFields.EVENT_ID, initialEventId)
                     .findFirst()
 
             shouldFetchInitialEvent = initialEvent == null
-            initialEvent?.root?.displayIndex
+            initialEvent?.displayIndex
         }
         prevDisplayIndex = initialDisplayIndex
         nextDisplayIndex = initialDisplayIndex
         if (currentInitialEventId != null && shouldFetchInitialEvent) {
             fetchEvent(currentInitialEventId)
         } else {
-            val count = min(settings.initialSize, filteredEvents.size)
+            val count = filteredEvents.size.coerceAtMost(settings.initialSize)
             if (initialEventId == null) {
-                paginateInternal(initialDisplayIndex, Timeline.Direction.BACKWARDS, count, strict = false)
+                paginateInternal(initialDisplayIndex, Timeline.Direction.BACKWARDS, count)
             } else {
-                paginateInternal(initialDisplayIndex, Timeline.Direction.FORWARDS, count / 2, strict = false)
-                paginateInternal(initialDisplayIndex, Timeline.Direction.BACKWARDS, count / 2, strict = true)
+                paginateInternal(initialDisplayIndex, Timeline.Direction.FORWARDS, (count / 2).coerceAtLeast(1))
+                paginateInternal(initialDisplayIndex?.minus(1), Timeline.Direction.BACKWARDS, (count / 2).coerceAtLeast(1))
             }
         }
         postSnapshot()
@@ -437,7 +480,7 @@ internal class DefaultTimeline(
     /**
      * This has to be called on TimelineThread as it access realm live results
      */
-    private fun handleUpdates(changeSet: OrderedCollectionChangeSet) {
+    private fun handleUpdates(results: RealmResults<TimelineEventEntity>, changeSet: OrderedCollectionChangeSet) {
         // If changeSet has deletion we are having a gap, so we clear everything
         if (changeSet.deletionRanges.isNotEmpty()) {
             clearAllValues()
@@ -445,9 +488,9 @@ internal class DefaultTimeline(
         var postSnapshot = false
         changeSet.insertionRanges.forEach { range ->
             val (startDisplayIndex, direction) = if (range.startIndex == 0) {
-                Pair(filteredEvents[range.length - 1]!!.root!!.displayIndex, Timeline.Direction.FORWARDS)
+                Pair(results[range.length - 1]!!.displayIndex, Timeline.Direction.FORWARDS)
             } else {
-                Pair(filteredEvents[range.startIndex]!!.root!!.displayIndex, Timeline.Direction.BACKWARDS)
+                Pair(results[range.startIndex]!!.displayIndex, Timeline.Direction.BACKWARDS)
             }
             val state = getState(direction)
             if (state.isPaginating) {
@@ -460,7 +503,7 @@ internal class DefaultTimeline(
             }
         }
         changeSet.changes.forEach { index ->
-            val eventEntity = filteredEvents[index]
+            val eventEntity = results[index]
             eventEntity?.eventId?.let { eventId ->
                 postSnapshot = rebuildEvent(eventId) {
                     buildTimelineEvent(eventEntity)
@@ -489,14 +532,13 @@ internal class DefaultTimeline(
         Timber.v("Should fetch $limit items $direction")
         cancelableBag += paginationTask
                 .configureWith(params) {
-                    this.constraints = TaskConstraints(connectedToNetwork = true)
                     this.callback = object : MatrixCallback<TokenChunkEventPersistor.Result> {
                         override fun onSuccess(data: TokenChunkEventPersistor.Result) {
                             when (data) {
-                                TokenChunkEventPersistor.Result.SUCCESS -> {
+                                TokenChunkEventPersistor.Result.SUCCESS           -> {
                                     Timber.v("Success fetching $limit items $direction from pagination request")
                                 }
-                                TokenChunkEventPersistor.Result.REACHED_END -> {
+                                TokenChunkEventPersistor.Result.REACHED_END       -> {
                                     postSnapshot()
                                 }
                                 TokenChunkEventPersistor.Result.SHOULD_FETCH_MORE ->
@@ -530,7 +572,7 @@ internal class DefaultTimeline(
      * This has to be called on TimelineThread as it access realm live results
      */
     private fun getLiveChunk(): ChunkEntity? {
-        return filteredEvents.firstOrNull()?.chunk?.firstOrNull()
+        return nonFilteredEvents.firstOrNull()?.chunk?.firstOrNull()
     }
 
     /**
@@ -539,17 +581,16 @@ internal class DefaultTimeline(
      */
     private fun buildTimelineEvents(startDisplayIndex: Int?,
                                     direction: Timeline.Direction,
-                                    count: Long,
-                                    strict: Boolean = false): Int {
+                                    count: Long): Int {
         if (count < 1 || startDisplayIndex == null) {
             return 0
         }
         val start = System.currentTimeMillis()
-        val offsetResults = getOffsetResults(startDisplayIndex, direction, count, strict)
+        val offsetResults = getOffsetResults(startDisplayIndex, direction, count)
         if (offsetResults.isEmpty()) {
             return 0
         }
-        val offsetIndex = offsetResults.last()!!.root!!.displayIndex
+        val offsetIndex = offsetResults.last()!!.displayIndex
         if (direction == Timeline.Direction.BACKWARDS) {
             prevDisplayIndex = offsetIndex - 1
         } else {
@@ -558,6 +599,11 @@ internal class DefaultTimeline(
         offsetResults.forEach { eventEntity ->
 
             val timelineEvent = buildTimelineEvent(eventEntity)
+            val transactionId = timelineEvent.root.unsignedData?.transactionId
+            val sendingEvent = inMemorySendingEvents.find {
+                it.eventId == transactionId
+            }
+            inMemorySendingEvents.remove(sendingEvent)
 
             if (timelineEvent.isEncrypted()
                     && timelineEvent.root.mxDecryptionResult == null) {
@@ -586,23 +632,16 @@ internal class DefaultTimeline(
      */
     private fun getOffsetResults(startDisplayIndex: Int,
                                  direction: Timeline.Direction,
-                                 count: Long,
-                                 strict: Boolean): RealmResults<TimelineEventEntity> {
+                                 count: Long): RealmResults<TimelineEventEntity> {
         val offsetQuery = filteredEvents.where()
         if (direction == Timeline.Direction.BACKWARDS) {
-            offsetQuery.sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.DESCENDING)
-            if (strict) {
-                offsetQuery.lessThan(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, startDisplayIndex)
-            } else {
-                offsetQuery.lessThanOrEqualTo(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, startDisplayIndex)
-            }
+            offsetQuery
+                    .sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING)
+                    .lessThanOrEqualTo(TimelineEventEntityFields.DISPLAY_INDEX, startDisplayIndex)
         } else {
-            offsetQuery.sort(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, Sort.ASCENDING)
-            if (strict) {
-                offsetQuery.greaterThan(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, startDisplayIndex)
-            } else {
-                offsetQuery.greaterThanOrEqualTo(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, startDisplayIndex)
-            }
+            offsetQuery
+                    .sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.ASCENDING)
+                    .greaterThanOrEqualTo(TimelineEventEntityFields.DISPLAY_INDEX, startDisplayIndex)
         }
         return offsetQuery
                 .limit(count)
@@ -612,19 +651,23 @@ internal class DefaultTimeline(
     private fun buildEventQuery(realm: Realm): RealmQuery<TimelineEventEntity> {
         return if (initialEventId == null) {
             TimelineEventEntity
-                    .where(realm, roomId = roomId, linkFilterMode = EventEntity.LinkFilterMode.LINKED_ONLY)
+                    .whereRoomId(realm, roomId = roomId)
                     .equalTo("${TimelineEventEntityFields.CHUNK}.${ChunkEntityFields.IS_LAST_FORWARD}", true)
         } else {
             TimelineEventEntity
-                    .where(realm, roomId = roomId, linkFilterMode = EventEntity.LinkFilterMode.BOTH)
+                    .whereRoomId(realm, roomId = roomId)
                     .`in`("${TimelineEventEntityFields.CHUNK}.${ChunkEntityFields.TIMELINE_EVENTS.EVENT_ID}", arrayOf(initialEventId))
         }
     }
 
     private fun fetchEvent(eventId: String) {
-        val params = GetContextOfEventTask.Params(roomId, eventId, settings.initialSize)
+        val params = GetContextOfEventTask.Params(roomId, eventId)
         cancelableBag += contextOfEventTask.configureWith(params) {
             callback = object : MatrixCallback<TokenChunkEventPersistor.Result> {
+                override fun onSuccess(data: TokenChunkEventPersistor.Result) {
+                    postSnapshot()
+                }
+
                 override fun onFailure(failure: Throwable) {
                     postFailure(failure)
                 }
@@ -645,7 +688,7 @@ internal class DefaultTimeline(
                     it.onTimelineUpdated(snapshot)
                 }
             }
-            debouncer.debounce("post_snapshot", runnable, 50)
+            debouncer.debounce("post_snapshot", runnable, 1)
         }
     }
 
@@ -670,7 +713,7 @@ internal class DefaultTimeline(
         forwardsState.set(State())
     }
 
-    // Extension methods ***************************************************************************
+// Extension methods ***************************************************************************
 
     private fun Timeline.Direction.toPaginationDirection(): PaginationDirection {
         return if (this == Timeline.Direction.BACKWARDS) PaginationDirection.BACKWARDS else PaginationDirection.FORWARDS
@@ -686,6 +729,23 @@ internal class DefaultTimeline(
         return this
     }
 
+    private fun List<TimelineEvent>.filterEventsWithSettings(): List<TimelineEvent> {
+        return filter {
+            val filterType = if (settings.filterTypes) {
+                settings.allowedTypes.contains(it.root.type)
+            } else {
+                true
+            }
+            val filterEdits = if (settings.filterEdits && it.root.type == EventType.MESSAGE) {
+                val messageContent = it.root.content.toModel<MessageContent>()
+                messageContent?.relatesTo?.type != RelationType.REPLACE
+            } else {
+                true
+            }
+            filterType && filterEdits
+        }
+    }
+
     private data class State(
             val hasReachedEnd: Boolean = false,
             val hasMoreInCache: Boolean = true,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultTimelineService.kt

@@ -34,16 +34,17 @@ import im.vector.matrix.android.internal.database.model.TimelineEventEntity
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.task.TaskExecutor
 import im.vector.matrix.android.internal.util.fetchCopyMap
+import org.greenrobot.eventbus.EventBus
 
 internal class DefaultTimelineService @AssistedInject constructor(@Assisted private val roomId: String,
                                                                   private val monarchy: Monarchy,
+                                                                  private val eventBus: EventBus,
                                                                   private val taskExecutor: TaskExecutor,
                                                                   private val contextOfEventTask: GetContextOfEventTask,
                                                                   private val cryptoService: CryptoService,
                                                                   private val paginationTask: PaginationTask,
                                                                   private val timelineEventMapper: TimelineEventMapper,
-                                                                  private val readReceiptsSummaryMapper: ReadReceiptsSummaryMapper,
-                                                                  private val clearUnlinkedEventsTask: ClearUnlinkedEventsTask
+                                                                  private val readReceiptsSummaryMapper: ReadReceiptsSummaryMapper
 ) : TimelineService {
 
     @AssistedInject.Factory
@@ -52,17 +53,18 @@ internal class DefaultTimelineService @AssistedInject constructor(@Assisted priv
     }
 
     override fun createTimeline(eventId: String?, settings: TimelineSettings): Timeline {
-        return DefaultTimeline(roomId,
-                eventId,
-                monarchy.realmConfiguration,
-                taskExecutor,
-                contextOfEventTask,
-                clearUnlinkedEventsTask,
-                paginationTask,
-                cryptoService,
-                timelineEventMapper,
-                settings,
-                TimelineHiddenReadReceipts(readReceiptsSummaryMapper, roomId, settings)
+        return DefaultTimeline(
+                roomId = roomId,
+                initialEventId = eventId,
+                realmConfiguration = monarchy.realmConfiguration,
+                taskExecutor = taskExecutor,
+                contextOfEventTask = contextOfEventTask,
+                paginationTask = paginationTask,
+                cryptoService = cryptoService,
+                timelineEventMapper = timelineEventMapper,
+                settings = settings,
+                hiddenReadReceipts = TimelineHiddenReadReceipts(readReceiptsSummaryMapper, roomId, settings),
+                eventBus = eventBus
         )
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TimelineEventDecryptor.kt

@@ -110,7 +110,7 @@ internal class TimelineEventDecryptor(
                 eventEntity.setDecryptionResult(result)
             }
         } catch (e: MXCryptoError) {
-            Timber.v(e, "Failed to decrypt event $eventId")
+            Timber.w(e, "Failed to decrypt event $eventId")
             if (e is MXCryptoError.Base && e.errorType == MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID) {
                 // Keep track of unknown sessions to automatically try to decrypt on new session
                 realm.executeTransaction {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TimelineHiddenReadReceipts.kt

@@ -76,12 +76,12 @@ internal class TimelineHiddenReadReceipts constructor(private val readReceiptsSu
             val timelineEvent = summary.timelineEvent?.firstOrNull() ?: continue
             val isLoaded = nonFilteredEvents.where()
                     .equalTo(TimelineEventEntityFields.EVENT_ID, timelineEvent.eventId).findFirst() != null
-            val displayIndex = timelineEvent.root?.displayIndex
+            val displayIndex = timelineEvent.displayIndex
 
-            if (isLoaded && displayIndex != null) {
+            if (isLoaded) {
                 // Then we are looking for the first displayable event after the hidden one
                 val firstDisplayedEvent = filteredEvents.where()
-                        .lessThanOrEqualTo(TimelineEventEntityFields.ROOT.DISPLAY_INDEX, displayIndex)
+                        .lessThanOrEqualTo(TimelineEventEntityFields.DISPLAY_INDEX, displayIndex)
                         .findFirst()
 
                 // If we find one, we should

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TokenChunkEventPersistor.kt

@@ -17,23 +17,37 @@
 package im.vector.matrix.android.internal.session.room.timeline
 
 import com.zhuinden.monarchy.Monarchy
-import im.vector.matrix.android.internal.database.helper.*
+import im.vector.matrix.android.api.session.events.model.EventType
+import im.vector.matrix.android.api.session.events.model.toModel
+import im.vector.matrix.android.api.session.room.model.RoomMemberContent
+import im.vector.matrix.android.api.session.room.send.SendState
+import im.vector.matrix.android.internal.database.helper.addOrUpdate
+import im.vector.matrix.android.internal.database.helper.addStateEvent
+import im.vector.matrix.android.internal.database.helper.addTimelineEvent
+import im.vector.matrix.android.internal.database.helper.deleteOnCascade
+import im.vector.matrix.android.internal.database.helper.merge
+import im.vector.matrix.android.internal.database.mapper.toEntity
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
+import im.vector.matrix.android.internal.database.model.TimelineEventEntity
 import im.vector.matrix.android.internal.database.query.create
 import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findAllIncludingEvents
+import im.vector.matrix.android.internal.database.query.findLastLiveChunkFromRoom
+import im.vector.matrix.android.internal.database.query.getOrCreate
+import im.vector.matrix.android.internal.database.query.latestEvent
 import im.vector.matrix.android.internal.database.query.where
+import im.vector.matrix.android.internal.session.room.RoomSummaryUpdater
 import im.vector.matrix.android.internal.util.awaitTransaction
-import io.realm.kotlin.createObject
+import io.realm.Realm
 import timber.log.Timber
 import javax.inject.Inject
 
 /**
  * Insert Chunk in DB, and eventually merge with existing chunk event
  */
-internal class TokenChunkEventPersistor @Inject constructor(private val monarchy: Monarchy,
-                                                            private val timelineEventSenderVisitor: TimelineEventSenderVisitor) {
+internal class TokenChunkEventPersistor @Inject constructor(private val monarchy: Monarchy) {
 
     /**
      * <pre>
@@ -111,9 +125,6 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
                 .awaitTransaction { realm ->
                     Timber.v("Start persisting ${receivedChunk.events.size} events in $roomId towards $direction")
 
-                    val roomEntity = RoomEntity.where(realm, roomId).findFirst()
-                            ?: realm.createObject(roomId)
-
                     val nextToken: String?
                     val prevToken: String?
                     if (direction == PaginationDirection.FORWARDS) {
@@ -124,50 +135,23 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
                         prevToken = receivedChunk.end
                     }
 
-                    val shouldSkip = ChunkEntity.find(realm, roomId, nextToken = nextToken) != null
-                            || ChunkEntity.find(realm, roomId, prevToken = prevToken) != null
-
                     val prevChunk = ChunkEntity.find(realm, roomId, nextToken = prevToken)
                     val nextChunk = ChunkEntity.find(realm, roomId, prevToken = nextToken)
 
                     // The current chunk is the one we will keep all along the merge processChanges.
                     // We try to look for a chunk next to the token,
                     // otherwise we create a whole new one which is unlinked (not live)
-
-                    var currentChunk = if (direction == PaginationDirection.FORWARDS) {
+                    val currentChunk = if (direction == PaginationDirection.FORWARDS) {
                         prevChunk?.apply { this.nextToken = nextToken }
                     } else {
                         nextChunk?.apply { this.prevToken = prevToken }
                     }
-                            ?: ChunkEntity.create(realm, prevToken, nextToken, isUnlinked = true)
+                            ?: ChunkEntity.create(realm, prevToken, nextToken)
 
                     if (receivedChunk.events.isEmpty() && receivedChunk.end == receivedChunk.start) {
-                        Timber.v("Reach end of $roomId")
-                        currentChunk.isLastBackward = true
-                    } else if (!shouldSkip) {
-                        Timber.v("Add ${receivedChunk.events.size} events in chunk(${currentChunk.nextToken} | ${currentChunk.prevToken}")
-                        val timelineEvents = receivedChunk.events.mapNotNull {
-                            currentChunk.add(roomId, it, direction)
-                        }
-                        // Then we merge chunks if needed
-                        if (currentChunk != prevChunk && prevChunk != null) {
-                            currentChunk = handleMerge(roomEntity, direction, currentChunk, prevChunk)
-                        } else if (currentChunk != nextChunk && nextChunk != null) {
-                            currentChunk = handleMerge(roomEntity, direction, currentChunk, nextChunk)
-                        } else {
-                            val newEventIds = receivedChunk.events.mapNotNull { it.eventId }
-                            val overlappedChunks = ChunkEntity.findAllIncludingEvents(realm, newEventIds)
-                            overlappedChunks
-                                    .filter { it != currentChunk }
-                                    .forEach { overlapped ->
-                                        currentChunk = handleMerge(roomEntity, direction, currentChunk, overlapped)
-                                    }
-                        }
-                        roomEntity.addOrUpdate(currentChunk)
-                        for (stateEvent in receivedChunk.stateEvents) {
-                            roomEntity.addStateEvent(stateEvent, isUnlinked = currentChunk.isUnlinked)
-                        }
-                        timelineEventSenderVisitor.visit(timelineEvents)
+                        handleReachEnd(realm, roomId, direction, currentChunk)
+                    } else {
+                        handlePagination(realm, roomId, direction, receivedChunk, currentChunk)
                     }
                 }
         return if (receivedChunk.events.isEmpty()) {
@@ -181,22 +165,90 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
         }
     }
 
-    private fun handleMerge(roomEntity: RoomEntity,
-                            direction: PaginationDirection,
-                            currentChunk: ChunkEntity,
-                            otherChunk: ChunkEntity): ChunkEntity {
-        // We always merge the bottom chunk into top chunk, so we are always merging backwards
-        Timber.v("Merge ${currentChunk.prevToken} | ${currentChunk.nextToken} with ${otherChunk.prevToken} | ${otherChunk.nextToken}")
-        return if (direction == PaginationDirection.BACKWARDS && !otherChunk.isLastForward) {
-            val events = currentChunk.merge(roomEntity.roomId, otherChunk, PaginationDirection.BACKWARDS)
-            timelineEventSenderVisitor.visit(events)
-            roomEntity.deleteOnCascade(otherChunk)
-            currentChunk
+    private fun handleReachEnd(realm: Realm, roomId: String, direction: PaginationDirection, currentChunk: ChunkEntity) {
+        Timber.v("Reach end of $roomId")
+        if (direction == PaginationDirection.FORWARDS) {
+            val currentLiveChunk = ChunkEntity.findLastLiveChunkFromRoom(realm, roomId)
+            if (currentChunk != currentLiveChunk) {
+                currentChunk.isLastForward = true
+                currentLiveChunk?.deleteOnCascade()
+                RoomSummaryEntity.where(realm, roomId).findFirst()?.apply {
+                    latestPreviewableEvent = TimelineEventEntity.latestEvent(
+                            realm,
+                            roomId,
+                            includesSending = true,
+                            filterTypes = RoomSummaryUpdater.PREVIEWABLE_TYPES
+                    )
+                }
+            }
         } else {
-            val events = otherChunk.merge(roomEntity.roomId, currentChunk, PaginationDirection.BACKWARDS)
-            timelineEventSenderVisitor.visit(events)
-            roomEntity.deleteOnCascade(currentChunk)
-            otherChunk
+            currentChunk.isLastBackward = true
         }
     }
+
+    private fun handlePagination(
+            realm: Realm,
+            roomId: String,
+            direction: PaginationDirection,
+            receivedChunk: TokenChunkEvent,
+            currentChunk: ChunkEntity
+    ) {
+        Timber.v("Add ${receivedChunk.events.size} events in chunk(${currentChunk.nextToken} | ${currentChunk.prevToken}")
+        val roomMemberContentsByUser = HashMap<String, RoomMemberContent?>()
+        val eventList = receivedChunk.events
+        val stateEvents = receivedChunk.stateEvents
+
+        for (stateEvent in stateEvents) {
+            val stateEventEntity = stateEvent.toEntity(roomId, SendState.SYNCED).let {
+                realm.copyToRealmOrUpdate(it)
+            }
+            currentChunk.addStateEvent(roomId, stateEventEntity, direction)
+            if (stateEvent.type == EventType.STATE_ROOM_MEMBER && stateEvent.stateKey != null) {
+                roomMemberContentsByUser[stateEvent.stateKey] = stateEvent.content.toModel<RoomMemberContent>()
+            }
+        }
+        val eventIds = ArrayList<String>(eventList.size)
+        for (event in eventList) {
+            if (event.eventId == null || event.senderId == null) {
+                continue
+            }
+            eventIds.add(event.eventId)
+            val eventEntity = event.toEntity(roomId, SendState.SYNCED).let {
+                realm.copyToRealmOrUpdate(it)
+            }
+            if (event.type == EventType.STATE_ROOM_MEMBER && event.stateKey != null) {
+                val contentToUse = if (direction == PaginationDirection.BACKWARDS) {
+                    event.prevContent
+                } else {
+                    event.content
+                }
+                roomMemberContentsByUser[event.stateKey] = contentToUse.toModel<RoomMemberContent>()
+            }
+
+            currentChunk.addTimelineEvent(roomId, eventEntity, direction, roomMemberContentsByUser)
+        }
+        val chunks = ChunkEntity.findAllIncludingEvents(realm, eventIds)
+        val chunksToDelete = ArrayList<ChunkEntity>()
+        chunks.forEach {
+            if (it != currentChunk) {
+                currentChunk.merge(roomId, it, direction)
+                chunksToDelete.add(it)
+            }
+        }
+        val shouldUpdateSummary = chunksToDelete.isNotEmpty() && currentChunk.isLastForward && direction == PaginationDirection.FORWARDS
+        chunksToDelete.forEach {
+            it.deleteOnCascade()
+        }
+        if (shouldUpdateSummary) {
+            val roomSummaryEntity = RoomSummaryEntity.getOrCreate(realm, roomId)
+            val latestPreviewableEvent = TimelineEventEntity.latestEvent(
+                    realm,
+                    roomId,
+                    includesSending = true,
+                    filterTypes = RoomSummaryUpdater.PREVIEWABLE_TYPES
+            )
+            roomSummaryEntity.latestPreviewableEvent = latestPreviewableEvent
+        }
+        RoomEntity.where(realm, roomId).findFirst()?.addOrUpdate(currentChunk)
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/tombstone/RoomTombstoneEventLiveObserver.kt

@@ -27,7 +27,7 @@ import im.vector.matrix.android.internal.database.awaitTransaction
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
@@ -41,7 +41,7 @@ internal class RoomTombstoneEventLiveObserver @Inject constructor(@SessionDataba
     : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(EventType.STATE_ROOM_TOMBSTONE))
+        EventEntity.whereTypes(it, listOf(EventType.STATE_ROOM_TOMBSTONE))
     }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/CryptoSyncHandler.kt

@@ -41,7 +41,7 @@ internal class CryptoSyncHandler @Inject constructor(private val cryptoService:
             // Decrypt event if necessary
             decryptEvent(event, null)
             if (event.getClearType() == EventType.MESSAGE
-                    && event.getClearContent()?.toModel<MessageContent>()?.type == "m.bad.encrypted") {
+                    && event.getClearContent()?.toModel<MessageContent>()?.msgType == "m.bad.encrypted") {
                 Timber.e("## handleToDeviceEvent() : Warning: Unable to decrypt to-device event : ${event.content}")
             } else {
                 verificationService.onToDeviceEvent(event)