Merge pull request #2976 from vector-im/feature/bma/local_echo

Snappier local echo

CHANGES.md

@@ -15,6 +15,7 @@ Improvements 🙌:
 Bugfix 🐛:
  - Try to fix crash about UrlPreview (#2640)
  - Be robust if Event.type is missing (#2946)
+ - Snappier message send status
 
 Translations 🗣:
  - All string resources and translations have been moved to the application module. Weblate project for the SDK will be removed.

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt

@@ -98,6 +98,19 @@ data class Event(
     @Transient
     var ageLocalTs: Long? = null
 
+    /**
+     * Copy all fields, including transient fields
+     */
+    fun copyAll(): Event {
+        return copy().also {
+            it.mxDecryptionResult = mxDecryptionResult
+            it.mCryptoError = mCryptoError
+            it.mCryptoErrorReason = mCryptoErrorReason
+            it.sendState = sendState
+            it.ageLocalTs = ageLocalTs
+        }
+    }
+
     /**
      * Check if event is a state event.
      * @return true if event is state event.

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/timeline/TimelineEvent.kt

@@ -38,6 +38,9 @@ import org.matrix.android.sdk.api.util.ContentUtils.extractUsefulTextFromReply
  */
 data class TimelineEvent(
         val root: Event,
+        /**
+         * Uniquely identify an event, computed locally by the sdk
+         */
         val localId: Long,
         val eventId: String,
         val displayIndex: Int,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoRepository.kt

@@ -27,7 +27,6 @@ import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
 import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.asyncTransaction
-import org.matrix.android.sdk.internal.database.helper.nextId
 import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
 import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.mapper.toEntity
@@ -45,6 +44,7 @@ import org.matrix.android.sdk.internal.session.room.timeline.TimelineInput
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.awaitTransaction
 import timber.log.Timber
+import java.util.UUID
 import javax.inject.Inject
 
 internal class LocalEchoRepository @Inject constructor(@SessionDatabase private val monarchy: Monarchy,
@@ -64,7 +64,7 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
             val eventEntity = event.toEntity(roomId, SendState.UNSENT, System.currentTimeMillis())
             val roomMemberHelper = RoomMemberHelper(realm, roomId)
             val myUser = roomMemberHelper.getLastRoomMember(senderId)
-            val localId = TimelineEventEntity.nextId(realm)
+            val localId = UUID.randomUUID().mostSignificantBits
             TimelineEventEntity(localId).also {
                 it.root = eventEntity
                 it.eventId = event.eventId

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/DefaultTimeline.kt

@@ -28,13 +28,7 @@ import org.matrix.android.sdk.api.NoOpMatrixCallback
 import org.matrix.android.sdk.api.extensions.orFalse
 import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.RelationType
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.api.session.room.model.EventAnnotationsSummary
-import org.matrix.android.sdk.api.session.room.model.ReactionAggregatedSummary
 import org.matrix.android.sdk.api.session.room.model.ReadReceipt
-import org.matrix.android.sdk.api.session.room.model.message.MessageContent
-import org.matrix.android.sdk.api.session.room.model.relation.ReactionContent
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.api.session.room.timeline.Timeline
 import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
@@ -46,7 +40,6 @@ import org.matrix.android.sdk.internal.database.model.ChunkEntity
 import org.matrix.android.sdk.internal.database.model.RoomEntity
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
-import org.matrix.android.sdk.internal.database.query.filterEvents
 import org.matrix.android.sdk.internal.database.query.findAllInRoomWithSendStates
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.database.query.whereRoomId
@@ -83,7 +76,8 @@ internal class DefaultTimeline(
         private val loadRoomMembersTask: LoadRoomMembersTask
 ) : Timeline,
         TimelineHiddenReadReceipts.Delegate,
-        TimelineInput.Listener {
+        TimelineInput.Listener,
+        UIEchoManager.Listener {
 
     companion object {
         val BACKGROUND_HANDLER = createBackgroundHandler("TIMELINE_DB_THREAD")
@@ -104,12 +98,12 @@ internal class DefaultTimeline(
     private var prevDisplayIndex: Int? = null
     private var nextDisplayIndex: Int? = null
 
-    private val uiEchoManager = UIEchoManager()
+    private val uiEchoManager = UIEchoManager(settings, this)
 
     private val builtEvents = Collections.synchronizedList<TimelineEvent>(ArrayList())
     private val builtEventsIdMap = Collections.synchronizedMap(HashMap<String, Int>())
-    private val backwardsState = AtomicReference(State())
-    private val forwardsState = AtomicReference(State())
+    private val backwardsState = AtomicReference(TimelineState())
+    private val forwardsState = AtomicReference(TimelineState())
 
     override val timelineID = UUID.randomUUID().toString()
 
@@ -168,13 +162,13 @@ internal class DefaultTimeline(
                 // are still used for ui echo (relation like reaction)
                 sendingEvents = roomEntity.sendingTimelineEvents.where()/*.filterEventsWithSettings()*/.findAll()
                 sendingEvents.addChangeListener { events ->
-                    uiEchoManager.sentEventsUpdated(events)
+                    uiEchoManager.onSentEventsInDatabase(events.map { it.eventId })
                     postSnapshot()
                 }
 
                 nonFilteredEvents = buildEventQuery(realm).sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING).findAll()
                 filteredEvents = nonFilteredEvents.where()
-                        .filterEventsWithSettings()
+                        .filterEventsWithSettings(settings)
                         .findAll()
                 nonFilteredEvents.addChangeListener(eventsChangeListener)
                 handleInitialLoad()
@@ -260,7 +254,9 @@ internal class DefaultTimeline(
                     .equalTo(TimelineEventEntityFields.EVENT_ID, eventId)
                     .findFirst()
 
-            val filteredEvents = nonFilteredEvents.where().filterEventsWithSettings().findAll()
+            val filteredEvents = nonFilteredEvents.where()
+                    .filterEventsWithSettings(settings)
+                    .findAll()
             val isEventInDb = nonFilteredEvent != null
 
             val isHidden = isEventInDb && filteredEvents.where()
@@ -326,20 +322,29 @@ internal class DefaultTimeline(
     }
 
     override fun onLocalEchoCreated(roomId: String, timelineEvent: TimelineEvent) {
-        if (uiEchoManager.onLocalEchoCreated(roomId, timelineEvent)) {
+        if (roomId != this.roomId || !isLive) return
+
+        val postSnapShot = uiEchoManager.onLocalEchoCreated(timelineEvent)
+
+        if (listOf(timelineEvent).filterEventsWithSettings(settings).isNotEmpty()) {
+            listeners.forEach {
+                it.onNewTimelineEvents(listOf(timelineEvent.eventId))
+            }
+        }
+
+        if (postSnapShot) {
             postSnapshot()
         }
     }
 
     override fun onLocalEchoUpdated(roomId: String, eventId: String, sendState: SendState) {
-        if (uiEchoManager.onLocalEchoUpdated(roomId, eventId, sendState)) {
+        if (roomId != this.roomId || !isLive) return
+        if (uiEchoManager.onSendStateUpdated(eventId, sendState)) {
             postSnapshot()
         }
     }
 
-// Private methods *****************************************************************************
-
-    private fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent?): Boolean {
+    override fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent?): Boolean {
         return tryOrNull {
             builtEventsIdMap[eventId]?.let { builtIndex ->
                 // Update the relation of existing event
@@ -359,6 +364,8 @@ internal class DefaultTimeline(
         } ?: false
     }
 
+// Private methods *****************************************************************************
+
     private fun hasMoreInCache(direction: Timeline.Direction) = getState(direction).hasMoreInCache
 
     private fun hasReachedEnd(direction: Timeline.Direction) = getState(direction).hasReachedEnd
@@ -411,35 +418,41 @@ internal class DefaultTimeline(
     }
 
     private fun buildSendingEvents(): List<TimelineEvent> {
-        val builtSendingEvents = ArrayList<TimelineEvent>()
+        val builtSendingEvents = mutableListOf<TimelineEvent>()
         if (hasReachedEnd(Timeline.Direction.FORWARDS) && !hasMoreInCache(Timeline.Direction.FORWARDS)) {
-            builtSendingEvents.addAll(uiEchoManager.getInMemorySendingEvents().filterEventsWithSettings())
+            uiEchoManager.getInMemorySendingEvents()
+                    .filterSendingEventsTo(builtSendingEvents)
             sendingEvents
-                    .map { timelineEventMapper.map(it) }
-                    // Filter out sending event that are not displayable!
-                    .filterEventsWithSettings()
-                    .forEach { timelineEvent ->
-                        if (builtSendingEvents.find { it.eventId == timelineEvent.eventId } == null) {
-                            uiEchoManager.updateSentStateWithUiEcho(timelineEvent)
-                            builtSendingEvents.add(timelineEvent)
-                        }
+                    .filter { timelineEvent ->
+                        builtSendingEvents.none { it.eventId == timelineEvent.eventId }
                     }
+                    .map { timelineEventMapper.map(it) }
+                    .filterSendingEventsTo(builtSendingEvents)
         }
         return builtSendingEvents
     }
 
+    private fun List<TimelineEvent>.filterSendingEventsTo(target: MutableList<TimelineEvent>) {
+        target.addAll(
+                // Filter out sending event that are not displayable!
+                filterEventsWithSettings(settings)
+                        // Get most up to date send state (in memory)
+                        .map { uiEchoManager.updateSentStateWithUiEcho(it) }
+        )
+    }
+
     private fun canPaginate(direction: Timeline.Direction): Boolean {
         return isReady.get() && !getState(direction).isPaginating && hasMoreToLoad(direction)
     }
 
-    private fun getState(direction: Timeline.Direction): State {
+    private fun getState(direction: Timeline.Direction): TimelineState {
         return when (direction) {
             Timeline.Direction.FORWARDS  -> forwardsState.get()
             Timeline.Direction.BACKWARDS -> backwardsState.get()
         }
     }
 
-    private fun updateState(direction: Timeline.Direction, update: (State) -> State) {
+    private fun updateState(direction: Timeline.Direction, update: (TimelineState) -> TimelineState) {
         val stateReference = when (direction) {
             Timeline.Direction.FORWARDS  -> forwardsState
             Timeline.Direction.BACKWARDS -> backwardsState
@@ -511,7 +524,7 @@ internal class DefaultTimeline(
             eventEntity?.eventId?.let { eventId ->
                 postSnapshot = rebuildEvent(eventId) {
                     val builtEvent = buildTimelineEvent(eventEntity)
-                    listOf(builtEvent).filterEventsWithSettings().firstOrNull()
+                    listOf(builtEvent).filterEventsWithSettings(settings).firstOrNull()
                 } || postSnapshot
             }
         }
@@ -744,8 +757,8 @@ internal class DefaultTimeline(
         nextDisplayIndex = null
         builtEvents.clear()
         builtEventsIdMap.clear()
-        backwardsState.set(State())
-        forwardsState.set(State())
+        backwardsState.set(TimelineState())
+        forwardsState.set(TimelineState())
     }
 
     private fun createPaginationCallback(limit: Int, direction: Timeline.Direction): MatrixCallback<TokenChunkEventPersistor.Result> {
@@ -779,191 +792,4 @@ internal class DefaultTimeline(
     private fun Timeline.Direction.toPaginationDirection(): PaginationDirection {
         return if (this == Timeline.Direction.BACKWARDS) PaginationDirection.BACKWARDS else PaginationDirection.FORWARDS
     }
-
-    private fun RealmQuery<TimelineEventEntity>.filterEventsWithSettings(): RealmQuery<TimelineEventEntity> {
-        return filterEvents(settings.filters)
-    }
-
-    private fun List<TimelineEvent>.filterEventsWithSettings(): List<TimelineEvent> {
-        return filter { event ->
-            val filterType = !settings.filters.filterTypes
-                    || settings.filters.allowedTypes.any { it.eventType == event.root.type && (it.stateKey == null || it.stateKey == event.root.senderId) }
-            if (!filterType) return@filter false
-
-            val filterEdits = if (settings.filters.filterEdits && event.root.getClearType() == EventType.MESSAGE) {
-                val messageContent = event.root.getClearContent().toModel<MessageContent>()
-                messageContent?.relatesTo?.type != RelationType.REPLACE && messageContent?.relatesTo?.type != RelationType.RESPONSE
-            } else {
-                true
-            }
-            if (!filterEdits) return@filter false
-
-            val filterRedacted = settings.filters.filterRedacted && event.root.isRedacted()
-            !filterRedacted
-        }
-    }
-
-    private data class State(
-            val hasReachedEnd: Boolean = false,
-            val hasMoreInCache: Boolean = true,
-            val isPaginating: Boolean = false,
-            val requestedPaginationCount: Int = 0
-    )
-
-    private data class ReactionUiEchoData(
-            val localEchoId: String,
-            val reactedOnEventId: String,
-            val reaction: String
-    )
-
-    inner class UIEchoManager {
-
-        private val inMemorySendingEvents = Collections.synchronizedList<TimelineEvent>(ArrayList())
-
-        fun getInMemorySendingEvents(): List<TimelineEvent> {
-            return inMemorySendingEvents.toList()
-        }
-
-        /**
-         * Due to lag of DB updates, we keep some UI echo of some properties to update timeline faster
-         */
-        private val inMemorySendingStates = Collections.synchronizedMap<String, SendState>(HashMap())
-
-        private val inMemoryReactions = Collections.synchronizedMap<String, MutableList<ReactionUiEchoData>>(HashMap())
-
-        fun sentEventsUpdated(events: RealmResults<TimelineEventEntity>) {
-            // Remove in memory as soon as they are known by database
-            events.forEach { te ->
-                inMemorySendingEvents.removeAll { te.eventId == it.eventId }
-            }
-            inMemorySendingStates.keys.removeAll { key ->
-                events.find { it.eventId == key } == null
-            }
-
-            inMemoryReactions.forEach { (_, uiEchoData) ->
-                uiEchoData.removeAll { data ->
-                    // I remove the uiEcho, when the related event is not anymore in the sending list
-                    // (means that it is synced)!
-                    events.find { it.eventId == data.localEchoId } == null
-                }
-            }
-        }
-
-        fun onLocalEchoUpdated(roomId: String, eventId: String, sendState: SendState): Boolean {
-            if (isLive && roomId == this@DefaultTimeline.roomId) {
-                val existingState = inMemorySendingStates[eventId]
-                inMemorySendingStates[eventId] = sendState
-                if (existingState != sendState) {
-                    return true
-                }
-            }
-            return false
-        }
-
-        // return true if should update
-        fun onLocalEchoCreated(roomId: String, timelineEvent: TimelineEvent): Boolean {
-            var postSnapshot = false
-            if (isLive && roomId == this@DefaultTimeline.roomId) {
-                // Manage some ui echos (do it before filter because actual event could be filtered out)
-                when (timelineEvent.root.getClearType()) {
-                    EventType.REDACTION -> {
-                    }
-                    EventType.REACTION  -> {
-                        val content = timelineEvent.root.content?.toModel<ReactionContent>()
-                        if (RelationType.ANNOTATION == content?.relatesTo?.type) {
-                            val reaction = content.relatesTo.key
-                            val relatedEventID = content.relatesTo.eventId
-                            inMemoryReactions.getOrPut(relatedEventID) { mutableListOf() }
-                                    .add(
-                                            ReactionUiEchoData(
-                                                    localEchoId = timelineEvent.eventId,
-                                                    reactedOnEventId = relatedEventID,
-                                                    reaction = reaction
-                                            )
-                                    )
-                            postSnapshot = rebuildEvent(relatedEventID) {
-                                decorateEventWithReactionUiEcho(it)
-                            } || postSnapshot
-                        }
-                    }
-                }
-
-                // do not add events that would have been filtered
-                if (listOf(timelineEvent).filterEventsWithSettings().isNotEmpty()) {
-                    listeners.forEach {
-                        it.onNewTimelineEvents(listOf(timelineEvent.eventId))
-                    }
-                    Timber.v("On local echo created: ${timelineEvent.eventId}")
-                    inMemorySendingEvents.add(0, timelineEvent)
-                    postSnapshot = true
-                }
-            }
-            return postSnapshot
-        }
-
-        fun decorateEventWithReactionUiEcho(timelineEvent: TimelineEvent): TimelineEvent? {
-            val relatedEventID = timelineEvent.eventId
-            val contents = inMemoryReactions[relatedEventID] ?: return null
-
-            var existingAnnotationSummary = timelineEvent.annotations ?: EventAnnotationsSummary(
-                    relatedEventID
-            )
-            val updateReactions = existingAnnotationSummary.reactionsSummary.toMutableList()
-
-            contents.forEach { uiEchoReaction ->
-                val existing = updateReactions.firstOrNull { it.key == uiEchoReaction.reaction }
-                if (existing == null) {
-                    // just add the new key
-                    ReactionAggregatedSummary(
-                            key = uiEchoReaction.reaction,
-                            count = 1,
-                            addedByMe = true,
-                            firstTimestamp = System.currentTimeMillis(),
-                            sourceEvents = emptyList(),
-                            localEchoEvents = listOf(uiEchoReaction.localEchoId)
-                    ).let { updateReactions.add(it) }
-                } else {
-                    // update Existing Key
-                    if (!existing.localEchoEvents.contains(uiEchoReaction.localEchoId)) {
-                        updateReactions.remove(existing)
-                        // only update if echo is not yet there
-                        ReactionAggregatedSummary(
-                                key = existing.key,
-                                count = existing.count + 1,
-                                addedByMe = true,
-                                firstTimestamp = existing.firstTimestamp,
-                                sourceEvents = existing.sourceEvents,
-                                localEchoEvents = existing.localEchoEvents + uiEchoReaction.localEchoId
-
-                        ).let { updateReactions.add(it) }
-                    }
-                }
-            }
-
-            existingAnnotationSummary = existingAnnotationSummary.copy(
-                    reactionsSummary = updateReactions
-            )
-            return timelineEvent.copy(
-                    annotations = existingAnnotationSummary
-            )
-        }
-
-        fun updateSentStateWithUiEcho(element: TimelineEvent) {
-            inMemorySendingStates[element.eventId]?.let {
-                // Timber.v("## ${System.currentTimeMillis()} Send event refresh echo with live state ${it} from state ${element.root.sendState}")
-                element.root.sendState = element.root.sendState.takeIf { it == SendState.SENT } ?: it
-            }
-        }
-
-        fun onSyncedEvent(transactionId: String?) {
-            val sendingEvent = inMemorySendingEvents.find {
-                it.eventId == transactionId
-            }
-            inMemorySendingEvents.remove(sendingEvent)
-            // Is it too early to clear it? will be done when removed from sending anyway?
-            inMemoryReactions.forEach { (_, u) ->
-                u.filterNot { it.localEchoId == transactionId }
-            }
-        }
-    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/Extensions.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.timeline
+
+import io.realm.RealmQuery
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.RelationType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.room.model.message.MessageContent
+import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
+import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
+import org.matrix.android.sdk.internal.database.query.filterEvents
+
+internal fun RealmQuery<TimelineEventEntity>.filterEventsWithSettings(settings: TimelineSettings): RealmQuery<TimelineEventEntity> {
+    return filterEvents(settings.filters)
+}
+
+internal fun List<TimelineEvent>.filterEventsWithSettings(settings: TimelineSettings): List<TimelineEvent> {
+    return filter { event ->
+        val filterType = !settings.filters.filterTypes
+                || settings.filters.allowedTypes.any { it.eventType == event.root.type && (it.stateKey == null || it.stateKey == event.root.senderId) }
+        if (!filterType) return@filter false
+
+        val filterEdits = if (settings.filters.filterEdits && event.root.getClearType() == EventType.MESSAGE) {
+            val messageContent = event.root.getClearContent().toModel<MessageContent>()
+            messageContent?.relatesTo?.type != RelationType.REPLACE && messageContent?.relatesTo?.type != RelationType.RESPONSE
+        } else {
+            true
+        }
+        if (!filterEdits) return@filter false
+
+        val filterRedacted = settings.filters.filterRedacted && event.root.isRedacted()
+        !filterRedacted
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/ReactionUiEchoData.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.timeline
+
+internal data class ReactionUiEchoData(
+        val localEchoId: String,
+        val reactedOnEventId: String,
+        val reaction: String
+)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineState.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.timeline
+
+internal data class TimelineState(
+        val hasReachedEnd: Boolean = false,
+        val hasMoreInCache: Boolean = true,
+        val isPaginating: Boolean = false,
+        val requestedPaginationCount: Int = 0
+)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/UIEchoManager.kt

@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.timeline
+
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.RelationType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.room.model.EventAnnotationsSummary
+import org.matrix.android.sdk.api.session.room.model.ReactionAggregatedSummary
+import org.matrix.android.sdk.api.session.room.model.relation.ReactionContent
+import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
+import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
+import timber.log.Timber
+import java.util.Collections
+
+internal class UIEchoManager(
+        private val settings: TimelineSettings,
+        private val listener: Listener
+) {
+
+    interface Listener {
+        fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent?): Boolean
+    }
+
+    private val inMemorySendingEvents = Collections.synchronizedList<TimelineEvent>(ArrayList())
+
+    fun getInMemorySendingEvents(): List<TimelineEvent> {
+        return inMemorySendingEvents.toList()
+    }
+
+    /**
+     * Due to lag of DB updates, we keep some UI echo of some properties to update timeline faster
+     */
+    private val inMemorySendingStates = Collections.synchronizedMap<String, SendState>(HashMap())
+
+    private val inMemoryReactions = Collections.synchronizedMap<String, MutableList<ReactionUiEchoData>>(HashMap())
+
+    fun onSentEventsInDatabase(eventIds: List<String>) {
+        // Remove in memory as soon as they are known by database
+        eventIds.forEach { eventId ->
+            inMemorySendingEvents.removeAll { eventId == it.eventId }
+        }
+        inMemoryReactions.forEach { (_, uiEchoData) ->
+            uiEchoData.removeAll { data ->
+                // I remove the uiEcho, when the related event is not anymore in the sending list
+                // (means that it is synced)!
+                eventIds.find { it == data.localEchoId } == null
+            }
+        }
+    }
+
+    fun onSendStateUpdated(eventId: String, sendState: SendState): Boolean {
+        val existingState = inMemorySendingStates[eventId]
+        inMemorySendingStates[eventId] = sendState
+        return existingState != sendState
+    }
+
+    // return true if should update
+    fun onLocalEchoCreated(timelineEvent: TimelineEvent): Boolean {
+        var postSnapshot = false
+
+        // Manage some ui echos (do it before filter because actual event could be filtered out)
+        when (timelineEvent.root.getClearType()) {
+            EventType.REDACTION -> {
+            }
+            EventType.REACTION  -> {
+                val content = timelineEvent.root.content?.toModel<ReactionContent>()
+                if (RelationType.ANNOTATION == content?.relatesTo?.type) {
+                    val reaction = content.relatesTo.key
+                    val relatedEventID = content.relatesTo.eventId
+                    inMemoryReactions.getOrPut(relatedEventID) { mutableListOf() }
+                            .add(
+                                    ReactionUiEchoData(
+                                            localEchoId = timelineEvent.eventId,
+                                            reactedOnEventId = relatedEventID,
+                                            reaction = reaction
+                                    )
+                            )
+                    postSnapshot = listener.rebuildEvent(relatedEventID) {
+                        decorateEventWithReactionUiEcho(it)
+                    } || postSnapshot
+                }
+            }
+        }
+
+        // do not add events that would have been filtered
+        if (listOf(timelineEvent).filterEventsWithSettings(settings).isNotEmpty()) {
+            Timber.v("On local echo created: ${timelineEvent.eventId}")
+            inMemorySendingEvents.add(0, timelineEvent)
+            postSnapshot = true
+        }
+
+        return postSnapshot
+    }
+
+    fun decorateEventWithReactionUiEcho(timelineEvent: TimelineEvent): TimelineEvent? {
+        val relatedEventID = timelineEvent.eventId
+        val contents = inMemoryReactions[relatedEventID] ?: return null
+
+        var existingAnnotationSummary = timelineEvent.annotations ?: EventAnnotationsSummary(
+                relatedEventID
+        )
+        val updateReactions = existingAnnotationSummary.reactionsSummary.toMutableList()
+
+        contents.forEach { uiEchoReaction ->
+            val existing = updateReactions.firstOrNull { it.key == uiEchoReaction.reaction }
+            if (existing == null) {
+                // just add the new key
+                ReactionAggregatedSummary(
+                        key = uiEchoReaction.reaction,
+                        count = 1,
+                        addedByMe = true,
+                        firstTimestamp = System.currentTimeMillis(),
+                        sourceEvents = emptyList(),
+                        localEchoEvents = listOf(uiEchoReaction.localEchoId)
+                ).let { updateReactions.add(it) }
+            } else {
+                // update Existing Key
+                if (!existing.localEchoEvents.contains(uiEchoReaction.localEchoId)) {
+                    updateReactions.remove(existing)
+                    // only update if echo is not yet there
+                    ReactionAggregatedSummary(
+                            key = existing.key,
+                            count = existing.count + 1,
+                            addedByMe = true,
+                            firstTimestamp = existing.firstTimestamp,
+                            sourceEvents = existing.sourceEvents,
+                            localEchoEvents = existing.localEchoEvents + uiEchoReaction.localEchoId
+
+                    ).let { updateReactions.add(it) }
+                }
+            }
+        }
+
+        existingAnnotationSummary = existingAnnotationSummary.copy(
+                reactionsSummary = updateReactions
+        )
+        return timelineEvent.copy(
+                annotations = existingAnnotationSummary
+        )
+    }
+
+    fun updateSentStateWithUiEcho(timelineEvent: TimelineEvent): TimelineEvent {
+        if (timelineEvent.root.sendState.isSent()) return timelineEvent
+        val inMemoryState = inMemorySendingStates[timelineEvent.eventId] ?: return timelineEvent
+        // Timber.v("## ${System.currentTimeMillis()} Send event refresh echo with live state $inMemoryState from state ${element.root.sendState}")
+        return timelineEvent.copy(
+                root = timelineEvent.root.copyAll()
+                        .also { it.sendState = inMemoryState }
+        )
+    }
+
+    fun onSyncedEvent(transactionId: String?) {
+        val sendingEvent = inMemorySendingEvents.find {
+            it.eventId == transactionId
+        }
+        inMemorySendingEvents.remove(sendingEvent)
+        // Is it too early to clear it? will be done when removed from sending anyway?
+        inMemoryReactions.forEach { (_, u) ->
+            u.filterNot { it.localEchoId == transactionId }
+        }
+        inMemorySendingStates.remove(transactionId)
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt

@@ -68,8 +68,10 @@ internal class DefaultSyncTask @Inject constructor(
     private val workingDir = File(fileDirectory, "is")
     private val initialSyncStatusRepository: InitialSyncStatusRepository = FileInitialSyncStatusRepository(workingDir)
 
-    override suspend fun execute(params: SyncTask.Params) = syncTaskSequencer.post {
-        doSync(params)
+    override suspend fun execute(params: SyncTask.Params) {
+        syncTaskSequencer.post {
+            doSync(params)
+        }
     }
 
     private suspend fun doSync(params: SyncTask.Params) {

vector/src/main/java/im/vector/app/features/home/room/detail/timeline/TimelineEventController.kt

@@ -155,6 +155,7 @@ class TimelineEventController @Inject constructor(private val dateFormatter: Vec
             synchronized(modelCache) {
                 assertUpdateCallbacksAllowed()
                 (position until (position + count)).forEach {
+                    // Invalidate cache
                     modelCache[it] = null
                 }
                 requestModelBuild()
@@ -173,7 +174,7 @@ class TimelineEventController @Inject constructor(private val dateFormatter: Vec
         override fun onInserted(position: Int, count: Int) {
             synchronized(modelCache) {
                 assertUpdateCallbacksAllowed()
-                (0 until count).forEach {
+                repeat(count) {
                     modelCache.add(position, null)
                 }
                 requestModelBuild()
@@ -183,7 +184,7 @@ class TimelineEventController @Inject constructor(private val dateFormatter: Vec
         override fun onRemoved(position: Int, count: Int) {
             synchronized(modelCache) {
                 assertUpdateCallbacksAllowed()
-                (0 until count).forEach {
+                repeat(count) {
                     modelCache.removeAt(position)
                 }
                 requestModelBuild()
@@ -427,6 +428,7 @@ class TimelineEventController @Inject constructor(private val dateFormatter: Vec
             val formattedDayModel: DaySeparatorItem? = null
     ) {
         fun shouldTriggerBuild(): Boolean {
+            // Since those items can change when we paginate, force a re-build
             return mergedHeaderModel != null || formattedDayModel != null
         }
     }