Sync : clean a bit by introducing 2 moshi adapters

matrix-sdk-android/build.gradle

@@ -13,6 +13,7 @@ buildscript {
     }
 }
 
+
 android {
     compileSdkVersion 30
     testOptions.unitTests.includeAndroidResources = true
@@ -88,6 +89,7 @@ android {
             java.srcDirs += "src/sharedTest/java"
         }
     }
+
 }
 
 static def gitRevision() {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt

@@ -36,7 +36,7 @@ import org.matrix.android.sdk.internal.network.parsing.ForceToBooleanJsonAdapter
 import org.matrix.android.sdk.internal.network.parsing.RuntimeJsonAdapterFactory
 import org.matrix.android.sdk.internal.network.parsing.TlsVersionMoshiAdapter
 import org.matrix.android.sdk.internal.network.parsing.UriMoshiAdapter
-import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter
+import org.matrix.android.sdk.internal.session.sync.parsing.DefaultLazyRoomSyncJsonAdapter
 
 object MoshiProvider {
 
@@ -45,7 +45,8 @@ object MoshiProvider {
             .add(ForceToBooleanJsonAdapter())
             .add(CipherSuiteMoshiAdapter())
             .add(TlsVersionMoshiAdapter())
-            .add(LazyRoomSyncJsonAdapter())
+            // Use addLast here so we can inject a SplitLazyRoomSyncJsonAdapter later to override the default parsing.
+            .addLast(DefaultLazyRoomSyncJsonAdapter())
             .add(RuntimeJsonAdapterFactory.of(MessageContent::class.java, "msgtype", MessageDefaultContent::class.java)
                     .registerSubtype(MessageTextContent::class.java, MessageType.MSGTYPE_TEXT)
                     .registerSubtype(MessageNoticeContent::class.java, MessageType.MSGTYPE_NOTICE)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt

@@ -17,10 +17,7 @@
 package org.matrix.android.sdk.internal.session.sync
 
 import okhttp3.ResponseBody
-import okio.buffer
-import okio.source
 import org.matrix.android.sdk.R
-import org.matrix.android.sdk.internal.di.MoshiProvider
 import org.matrix.android.sdk.internal.di.SessionFilesDirectory
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.network.GlobalErrorReceiver
@@ -32,8 +29,8 @@ import org.matrix.android.sdk.internal.session.filter.FilterRepository
 import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask
 import org.matrix.android.sdk.internal.session.reportSubtask
 import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync
-import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter
 import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
+import org.matrix.android.sdk.internal.session.sync.parsing.InitialSyncResponseParser
 import org.matrix.android.sdk.internal.session.user.UserStore
 import org.matrix.android.sdk.internal.task.Task
 import org.matrix.android.sdk.internal.util.logDuration
@@ -64,7 +61,8 @@ internal class DefaultSyncTask @Inject constructor(
         private val syncTaskSequencer: SyncTaskSequencer,
         private val globalErrorReceiver: GlobalErrorReceiver,
         @SessionFilesDirectory
-        private val fileDirectory: File
+        private val fileDirectory: File,
+        private val syncResponseParser: InitialSyncResponseParser
 ) : SyncTask {
 
     private val workingDir = File(fileDirectory, "is")
@@ -101,9 +99,10 @@ internal class DefaultSyncTask @Inject constructor(
         val readTimeOut = (params.timeout + TIMEOUT_MARGIN).coerceAtLeast(TimeOutInterceptor.DEFAULT_LONG_TIMEOUT)
 
         if (isInitialSync) {
-            logDuration("INIT_SYNC strategy: $initialSyncStrategy") {
+            val initSyncStrategy = initialSyncStrategy
-                if (initialSyncStrategy is InitialSyncStrategy.Optimized) {
+            logDuration("INIT_SYNC strategy: $initSyncStrategy") {
-                    safeInitialSync(requestParams)
+                if (initSyncStrategy is InitialSyncStrategy.Optimized) {
+                    safeInitialSync(requestParams, initSyncStrategy)
                 } else {
                     val syncResponse = logDuration("INIT_SYNC Request") {
                         executeRequest<SyncResponse>(globalErrorReceiver) {
@@ -132,7 +131,7 @@ internal class DefaultSyncTask @Inject constructor(
         Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}")
     }
 
-    private suspend fun safeInitialSync(requestParams: Map<String, String>) {
+    private suspend fun safeInitialSync(requestParams: Map<String, String>, initSyncStrategy: InitialSyncStrategy.Optimized) {
         workingDir.mkdirs()
         val workingFile = File(workingDir, "initSync.json")
         val status = initialSyncStatusRepository.getStep()
@@ -163,7 +162,7 @@ internal class DefaultSyncTask @Inject constructor(
             }
             initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED)
         }
-        handleSyncFile(workingFile)
+        handleSyncFile(workingFile, initSyncStrategy)
 
         // Delete all files
         workingDir.deleteRecursively()
@@ -188,44 +187,22 @@ internal class DefaultSyncTask @Inject constructor(
         }
     }
 
-    private suspend fun handleSyncFile(workingFile: File) {
+    private suspend fun handleSyncFile(workingFile: File, initSyncStrategy: InitialSyncStrategy.Optimized) {
-        val syncResponseLength = workingFile.length().toInt()
+        logDuration("INIT_SYNC handleSyncFile()") {
-
+            val syncResponse = logDuration("INIT_SYNC Read file and parse") {
-        logDuration("INIT_SYNC handleSyncFile() file size $syncResponseLength bytes") {
+                syncResponseParser.parse(initSyncStrategy, workingFile)
-            if (syncResponseLength < (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToSplit ?: Long.MAX_VALUE) {
-                // OK, no need to split just handle as a regular sync response
-                Timber.v("INIT_SYNC no need to split")
-                handleInitialSyncFile(workingFile)
-            } else {
-                Timber.v("INIT_SYNC Split into several smaller files")
-                // Set file mode
-                // TODO This is really ugly, I should improve that
-                LazyRoomSyncJsonAdapter.initWith(workingFile)
-
-                handleInitialSyncFile(workingFile)
-
-                // Reset file mode
-                LazyRoomSyncJsonAdapter.reset()
             }
-        }
+            initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED)
-    }
+            // Log some stats
+            val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0
+            val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored }
+            Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files")
 
-    private suspend fun handleInitialSyncFile(workingFile: File) {
+            logDuration("INIT_SYNC Database insertion") {
-        val syncResponse = logDuration("INIT_SYNC Read file and parse") {
+                syncResponseHandler.handleResponse(syncResponse, null)
-            MoshiProvider.providesMoshi().adapter(SyncResponse::class.java)
+            }
-                    .fromJson(workingFile.source().buffer())!!
+            initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS)
         }
-        initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED)
-
-        // Log some stats
-        val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0
-        val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored }
-        Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files")
-
-        logDuration("INIT_SYNC Database insertion") {
-            syncResponseHandler.handleResponse(syncResponse, null)
-        }
-        initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS)
     }
 
     companion object {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt

@@ -16,17 +16,17 @@
 
 package org.matrix.android.sdk.internal.session.sync.model
 
+import com.squareup.moshi.JsonAdapter
 import com.squareup.moshi.JsonClass
 import com.squareup.moshi.JsonReader
 import okio.buffer
 import okio.source
-import org.matrix.android.sdk.internal.di.MoshiProvider
 import java.io.File
 
 @JsonClass(generateAdapter = false)
 internal sealed class LazyRoomSync {
     data class Parsed(val _roomSync: RoomSync) : LazyRoomSync()
-    data class Stored(val file: File) : LazyRoomSync()
+    data class Stored(val roomSyncAdapter: JsonAdapter<RoomSync>, val file: File) : LazyRoomSync()
 
     val roomSync: RoomSync
         get() {
@@ -35,8 +35,7 @@ internal sealed class LazyRoomSync {
                 is Stored -> {
                     // Parse the file now
                     file.inputStream().use { pos ->
-                        MoshiProvider.providesMoshi().adapter(RoomSync::class.java)
+                        roomSyncAdapter.fromJson(JsonReader.of(pos.source().buffer()))!!
-                                .fromJson(JsonReader.of(pos.source().buffer()))!!
                     }
                 }
             }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt

@@ -1,90 +0,0 @@
-/*
- * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.session.sync.model
-
-import com.squareup.moshi.FromJson
-import com.squareup.moshi.JsonAdapter
-import com.squareup.moshi.JsonReader
-import com.squareup.moshi.JsonWriter
-import com.squareup.moshi.ToJson
-import org.matrix.android.sdk.internal.di.MoshiProvider
-import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
-import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy
-import timber.log.Timber
-import java.io.File
-import java.util.concurrent.atomic.AtomicInteger
-
-internal class LazyRoomSyncJsonAdapter : JsonAdapter<LazyRoomSync>() {
-
-    @FromJson
-    override fun fromJson(reader: JsonReader): LazyRoomSync {
-        return if (workingDirectory != null) {
-            val path = reader.path
-            // val roomId = reader.path.substringAfter("\$.rooms.join.")
-
-            // inputStream.available() return 0... So read it to a String then decide to store in a file or to parse it now
-            val json = reader.nextSource().inputStream().bufferedReader().use {
-                it.readText()
-            }
-
-            val limit = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToStoreInFile ?: Long.MAX_VALUE
-            if (json.length > limit) {
-                Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file")
-                // Copy the source to a file
-                val file = createFile()
-                file.writeText(json)
-                LazyRoomSync.Stored(file)
-            } else {
-                Timber.v("INIT_SYNC $path content length: ${json.length} parse it now")
-                // Parse it now
-                val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(json)!!
-                LazyRoomSync.Parsed(roomSync)
-            }
-        } else {
-            // Parse it now
-            val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(reader)!!
-            LazyRoomSync.Parsed(roomSync)
-        }
-    }
-
-    @ToJson
-    override fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
-        // This Adapter is not supposed to serialize object
-        throw UnsupportedOperationException()
-    }
-
-    companion object {
-        fun initWith(file: File) {
-            workingDirectory = file.parentFile
-            atomicInteger.set(0)
-        }
-
-        fun reset() {
-            workingDirectory = null
-        }
-
-        private fun createFile(): File {
-            val parent = workingDirectory ?: error("workingDirectory is not initialized")
-            val index = atomicInteger.getAndIncrement()
-
-            return File(parent, "room_$index.json")
-        }
-
-        private var workingDirectory: File? = null
-        private val atomicInteger = AtomicInteger(0)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2021 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.sync.parsing
+
+import com.squareup.moshi.Moshi
+import okio.buffer
+import okio.source
+import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
+import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
+import timber.log.Timber
+import java.io.File
+import javax.inject.Inject
+
+internal class InitialSyncResponseParser @Inject constructor(private val moshi: Moshi) {
+
+    fun parse(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File): SyncResponse {
+        val syncResponseLength = workingFile.length().toInt()
+        Timber.v("Sync file size is $syncResponseLength bytes")
+        val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit
+        Timber.v("INIT_SYNC should split in several files: $shouldSplit")
+        return getMoshi(syncStrategy, workingFile, shouldSplit)
+                .adapter(SyncResponse::class.java)
+                .fromJson(workingFile.source().buffer())!!
+    }
+
+    private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File, shouldSplit: Boolean): Moshi {
+        // If we don't have to split we'll rely on the already default moshi
+        if (!shouldSplit) return moshi
+        // Otherwise, we create a new adapter for handling Map of Lazy sync
+        return moshi.newBuilder()
+                .add(SplitLazyRoomSyncJsonAdapter(workingFile, syncStrategy))
+                .build()
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2021 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.sync.parsing
+
+import com.squareup.moshi.FromJson
+import com.squareup.moshi.JsonAdapter
+import com.squareup.moshi.JsonReader
+import com.squareup.moshi.JsonWriter
+import com.squareup.moshi.ToJson
+import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
+import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync
+import org.matrix.android.sdk.internal.session.sync.model.RoomSync
+import timber.log.Timber
+import java.io.File
+import java.util.concurrent.atomic.AtomicInteger
+
+internal class DefaultLazyRoomSyncJsonAdapter() {
+
+    @FromJson
+    fun fromJson(reader: JsonReader, delegate: JsonAdapter<RoomSync>): LazyRoomSync? {
+        val roomSync = delegate.fromJson(reader) ?: return null
+        return LazyRoomSync.Parsed(roomSync)
+    }
+
+    @ToJson
+    fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
+        // This Adapter is not supposed to serialize object
+        Timber.v("To json $value with $writer")
+        throw UnsupportedOperationException()
+    }
+}
+
+internal class SplitLazyRoomSyncJsonAdapter(
+        private val workingDirectory: File,
+        private val syncStrategy: InitialSyncStrategy.Optimized
+) {
+
+    private val atomicInteger = AtomicInteger(0)
+
+    private fun createFile(): File {
+        val index = atomicInteger.getAndIncrement()
+        return File(workingDirectory.parentFile, "room_$index.json")
+    }
+
+    @FromJson
+    fun fromJson(reader: JsonReader, delegate: JsonAdapter<RoomSync>): LazyRoomSync? {
+        val path = reader.path
+        val json = reader.nextSource().inputStream().bufferedReader().use {
+            it.readText()
+        }
+        val limit = syncStrategy.minSizeToStoreInFile
+        return if (json.length > limit) {
+            Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file")
+            // Copy the source to a file
+            val file = createFile()
+            file.writeText(json)
+            LazyRoomSync.Stored(delegate, file)
+        } else {
+            Timber.v("INIT_SYNC $path content length: ${json.length} parse it now")
+            val roomSync = delegate.fromJson(json) ?: return null
+            LazyRoomSync.Parsed(roomSync)
+        }
+    }
+
+    @ToJson
+    fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
+        // This Adapter is not supposed to serialize object
+        Timber.v("To json $value with $writer")
+        throw UnsupportedOperationException()
+    }
+}