From 3633199e683cfc308167d8dda4e5cc3890724d57 Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 4 Feb 2021 14:01:21 +0100 Subject: [PATCH] Sync : clean a bit by introducing 2 moshi adapters --- matrix-sdk-android/build.gradle | 2 + .../android/sdk/internal/di/MoshiProvider.kt | 5 +- .../sdk/internal/session/sync/SyncTask.kt | 67 +++++--------- .../session/sync/model/LazyRoomSync.kt | 7 +- .../sync/model/LazyRoomSyncJsonAdapter.kt | 90 ------------------- .../sync/parsing/InitialSyncResponseParser.kt | 48 ++++++++++ .../sync/parsing/LazyRoomSyncJsonAdapters.kt | 85 ++++++++++++++++++ 7 files changed, 163 insertions(+), 141 deletions(-) delete mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index 3ade9f7214..4f8604bfc1 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -13,6 +13,7 @@ buildscript { } } + android { compileSdkVersion 30 testOptions.unitTests.includeAndroidResources = true @@ -88,6 +89,7 @@ android { java.srcDirs += "src/sharedTest/java" } } + } static def gitRevision() { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt index c0595a9273..b6bc17eaa0 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt @@ -36,7 +36,7 @@ import org.matrix.android.sdk.internal.network.parsing.ForceToBooleanJsonAdapter import org.matrix.android.sdk.internal.network.parsing.RuntimeJsonAdapterFactory import org.matrix.android.sdk.internal.network.parsing.TlsVersionMoshiAdapter import org.matrix.android.sdk.internal.network.parsing.UriMoshiAdapter -import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter +import org.matrix.android.sdk.internal.session.sync.parsing.DefaultLazyRoomSyncJsonAdapter object MoshiProvider { @@ -45,7 +45,8 @@ object MoshiProvider { .add(ForceToBooleanJsonAdapter()) .add(CipherSuiteMoshiAdapter()) .add(TlsVersionMoshiAdapter()) - .add(LazyRoomSyncJsonAdapter()) + // Use addLast here so we can inject a SplitLazyRoomSyncJsonAdapter later to override the default parsing. + .addLast(DefaultLazyRoomSyncJsonAdapter()) .add(RuntimeJsonAdapterFactory.of(MessageContent::class.java, "msgtype", MessageDefaultContent::class.java) .registerSubtype(MessageTextContent::class.java, MessageType.MSGTYPE_TEXT) .registerSubtype(MessageNoticeContent::class.java, MessageType.MSGTYPE_NOTICE) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index ef766f6d42..dde3da48c6 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -17,10 +17,7 @@ package org.matrix.android.sdk.internal.session.sync import okhttp3.ResponseBody -import okio.buffer -import okio.source import org.matrix.android.sdk.R -import org.matrix.android.sdk.internal.di.MoshiProvider import org.matrix.android.sdk.internal.di.SessionFilesDirectory import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.network.GlobalErrorReceiver @@ -32,8 +29,8 @@ import org.matrix.android.sdk.internal.session.filter.FilterRepository import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask import org.matrix.android.sdk.internal.session.reportSubtask import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync -import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter import org.matrix.android.sdk.internal.session.sync.model.SyncResponse +import org.matrix.android.sdk.internal.session.sync.parsing.InitialSyncResponseParser import org.matrix.android.sdk.internal.session.user.UserStore import org.matrix.android.sdk.internal.task.Task import org.matrix.android.sdk.internal.util.logDuration @@ -64,7 +61,8 @@ internal class DefaultSyncTask @Inject constructor( private val syncTaskSequencer: SyncTaskSequencer, private val globalErrorReceiver: GlobalErrorReceiver, @SessionFilesDirectory - private val fileDirectory: File + private val fileDirectory: File, + private val syncResponseParser: InitialSyncResponseParser ) : SyncTask { private val workingDir = File(fileDirectory, "is") @@ -101,9 +99,10 @@ internal class DefaultSyncTask @Inject constructor( val readTimeOut = (params.timeout + TIMEOUT_MARGIN).coerceAtLeast(TimeOutInterceptor.DEFAULT_LONG_TIMEOUT) if (isInitialSync) { - logDuration("INIT_SYNC strategy: $initialSyncStrategy") { - if (initialSyncStrategy is InitialSyncStrategy.Optimized) { - safeInitialSync(requestParams) + val initSyncStrategy = initialSyncStrategy + logDuration("INIT_SYNC strategy: $initSyncStrategy") { + if (initSyncStrategy is InitialSyncStrategy.Optimized) { + safeInitialSync(requestParams, initSyncStrategy) } else { val syncResponse = logDuration("INIT_SYNC Request") { executeRequest(globalErrorReceiver) { @@ -132,7 +131,7 @@ internal class DefaultSyncTask @Inject constructor( Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}") } - private suspend fun safeInitialSync(requestParams: Map) { + private suspend fun safeInitialSync(requestParams: Map, initSyncStrategy: InitialSyncStrategy.Optimized) { workingDir.mkdirs() val workingFile = File(workingDir, "initSync.json") val status = initialSyncStatusRepository.getStep() @@ -163,7 +162,7 @@ internal class DefaultSyncTask @Inject constructor( } initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED) } - handleSyncFile(workingFile) + handleSyncFile(workingFile, initSyncStrategy) // Delete all files workingDir.deleteRecursively() @@ -188,44 +187,22 @@ internal class DefaultSyncTask @Inject constructor( } } - private suspend fun handleSyncFile(workingFile: File) { - val syncResponseLength = workingFile.length().toInt() - - logDuration("INIT_SYNC handleSyncFile() file size $syncResponseLength bytes") { - if (syncResponseLength < (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToSplit ?: Long.MAX_VALUE) { - // OK, no need to split just handle as a regular sync response - Timber.v("INIT_SYNC no need to split") - handleInitialSyncFile(workingFile) - } else { - Timber.v("INIT_SYNC Split into several smaller files") - // Set file mode - // TODO This is really ugly, I should improve that - LazyRoomSyncJsonAdapter.initWith(workingFile) - - handleInitialSyncFile(workingFile) - - // Reset file mode - LazyRoomSyncJsonAdapter.reset() + private suspend fun handleSyncFile(workingFile: File, initSyncStrategy: InitialSyncStrategy.Optimized) { + logDuration("INIT_SYNC handleSyncFile()") { + val syncResponse = logDuration("INIT_SYNC Read file and parse") { + syncResponseParser.parse(initSyncStrategy, workingFile) } - } - } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED) + // Log some stats + val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0 + val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored } + Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") - private suspend fun handleInitialSyncFile(workingFile: File) { - val syncResponse = logDuration("INIT_SYNC Read file and parse") { - MoshiProvider.providesMoshi().adapter(SyncResponse::class.java) - .fromJson(workingFile.source().buffer())!! + logDuration("INIT_SYNC Database insertion") { + syncResponseHandler.handleResponse(syncResponse, null) + } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) } - initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED) - - // Log some stats - val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0 - val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored } - Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") - - logDuration("INIT_SYNC Database insertion") { - syncResponseHandler.handleResponse(syncResponse, null) - } - initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) } companion object { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt index e7e0dcb8b9..1f6e3de9a4 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt @@ -16,17 +16,17 @@ package org.matrix.android.sdk.internal.session.sync.model +import com.squareup.moshi.JsonAdapter import com.squareup.moshi.JsonClass import com.squareup.moshi.JsonReader import okio.buffer import okio.source -import org.matrix.android.sdk.internal.di.MoshiProvider import java.io.File @JsonClass(generateAdapter = false) internal sealed class LazyRoomSync { data class Parsed(val _roomSync: RoomSync) : LazyRoomSync() - data class Stored(val file: File) : LazyRoomSync() + data class Stored(val roomSyncAdapter: JsonAdapter, val file: File) : LazyRoomSync() val roomSync: RoomSync get() { @@ -35,8 +35,7 @@ internal sealed class LazyRoomSync { is Stored -> { // Parse the file now file.inputStream().use { pos -> - MoshiProvider.providesMoshi().adapter(RoomSync::class.java) - .fromJson(JsonReader.of(pos.source().buffer()))!! + roomSyncAdapter.fromJson(JsonReader.of(pos.source().buffer()))!! } } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt deleted file mode 100644 index edf76d2e77..0000000000 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021 The Matrix.org Foundation C.I.C. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.matrix.android.sdk.internal.session.sync.model - -import com.squareup.moshi.FromJson -import com.squareup.moshi.JsonAdapter -import com.squareup.moshi.JsonReader -import com.squareup.moshi.JsonWriter -import com.squareup.moshi.ToJson -import org.matrix.android.sdk.internal.di.MoshiProvider -import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy -import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy -import timber.log.Timber -import java.io.File -import java.util.concurrent.atomic.AtomicInteger - -internal class LazyRoomSyncJsonAdapter : JsonAdapter() { - - @FromJson - override fun fromJson(reader: JsonReader): LazyRoomSync { - return if (workingDirectory != null) { - val path = reader.path - // val roomId = reader.path.substringAfter("\$.rooms.join.") - - // inputStream.available() return 0... So read it to a String then decide to store in a file or to parse it now - val json = reader.nextSource().inputStream().bufferedReader().use { - it.readText() - } - - val limit = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToStoreInFile ?: Long.MAX_VALUE - if (json.length > limit) { - Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file") - // Copy the source to a file - val file = createFile() - file.writeText(json) - LazyRoomSync.Stored(file) - } else { - Timber.v("INIT_SYNC $path content length: ${json.length} parse it now") - // Parse it now - val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(json)!! - LazyRoomSync.Parsed(roomSync) - } - } else { - // Parse it now - val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(reader)!! - LazyRoomSync.Parsed(roomSync) - } - } - - @ToJson - override fun toJson(writer: JsonWriter, value: LazyRoomSync?) { - // This Adapter is not supposed to serialize object - throw UnsupportedOperationException() - } - - companion object { - fun initWith(file: File) { - workingDirectory = file.parentFile - atomicInteger.set(0) - } - - fun reset() { - workingDirectory = null - } - - private fun createFile(): File { - val parent = workingDirectory ?: error("workingDirectory is not initialized") - val index = atomicInteger.getAndIncrement() - - return File(parent, "room_$index.json") - } - - private var workingDirectory: File? = null - private val atomicInteger = AtomicInteger(0) - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt new file mode 100644 index 0000000000..760f25270f --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.parsing + +import com.squareup.moshi.Moshi +import okio.buffer +import okio.source +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.model.SyncResponse +import timber.log.Timber +import java.io.File +import javax.inject.Inject + +internal class InitialSyncResponseParser @Inject constructor(private val moshi: Moshi) { + + fun parse(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File): SyncResponse { + val syncResponseLength = workingFile.length().toInt() + Timber.v("Sync file size is $syncResponseLength bytes") + val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit + Timber.v("INIT_SYNC should split in several files: $shouldSplit") + return getMoshi(syncStrategy, workingFile, shouldSplit) + .adapter(SyncResponse::class.java) + .fromJson(workingFile.source().buffer())!! + } + + private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File, shouldSplit: Boolean): Moshi { + // If we don't have to split we'll rely on the already default moshi + if (!shouldSplit) return moshi + // Otherwise, we create a new adapter for handling Map of Lazy sync + return moshi.newBuilder() + .add(SplitLazyRoomSyncJsonAdapter(workingFile, syncStrategy)) + .build() + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt new file mode 100644 index 0000000000..232e2fe99e --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.parsing + +import com.squareup.moshi.FromJson +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.JsonReader +import com.squareup.moshi.JsonWriter +import com.squareup.moshi.ToJson +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync +import org.matrix.android.sdk.internal.session.sync.model.RoomSync +import timber.log.Timber +import java.io.File +import java.util.concurrent.atomic.AtomicInteger + +internal class DefaultLazyRoomSyncJsonAdapter() { + + @FromJson + fun fromJson(reader: JsonReader, delegate: JsonAdapter): LazyRoomSync? { + val roomSync = delegate.fromJson(reader) ?: return null + return LazyRoomSync.Parsed(roomSync) + } + + @ToJson + fun toJson(writer: JsonWriter, value: LazyRoomSync?) { + // This Adapter is not supposed to serialize object + Timber.v("To json $value with $writer") + throw UnsupportedOperationException() + } +} + +internal class SplitLazyRoomSyncJsonAdapter( + private val workingDirectory: File, + private val syncStrategy: InitialSyncStrategy.Optimized +) { + + private val atomicInteger = AtomicInteger(0) + + private fun createFile(): File { + val index = atomicInteger.getAndIncrement() + return File(workingDirectory.parentFile, "room_$index.json") + } + + @FromJson + fun fromJson(reader: JsonReader, delegate: JsonAdapter): LazyRoomSync? { + val path = reader.path + val json = reader.nextSource().inputStream().bufferedReader().use { + it.readText() + } + val limit = syncStrategy.minSizeToStoreInFile + return if (json.length > limit) { + Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file") + // Copy the source to a file + val file = createFile() + file.writeText(json) + LazyRoomSync.Stored(delegate, file) + } else { + Timber.v("INIT_SYNC $path content length: ${json.length} parse it now") + val roomSync = delegate.fromJson(json) ?: return null + LazyRoomSync.Parsed(roomSync) + } + } + + @ToJson + fun toJson(writer: JsonWriter, value: LazyRoomSync?) { + // This Adapter is not supposed to serialize object + Timber.v("To json $value with $writer") + throw UnsupportedOperationException() + } +}