mirror of
https://github.com/element-hq/element-android
synced 2024-11-24 02:15:35 +03:00
Sync : clean a bit by introducing 2 moshi adapters
This commit is contained in:
ganfra
Benoit Marty
parent
8e2161bd9e
commit
3633199e68
7 changed files with 163 additions and 141 deletions
|
|
@ -13,6 +13,7 @@ buildscript {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
android {
|
||||
compileSdkVersion 30
|
||||
testOptions.unitTests.includeAndroidResources = true
|
||||
|
|
@ -88,6 +89,7 @@ android {
|
|||
java.srcDirs += "src/sharedTest/java"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static def gitRevision() {
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ import org.matrix.android.sdk.internal.network.parsing.ForceToBooleanJsonAdapter
|
|||
import org.matrix.android.sdk.internal.network.parsing.RuntimeJsonAdapterFactory
|
||||
import org.matrix.android.sdk.internal.network.parsing.TlsVersionMoshiAdapter
|
||||
import org.matrix.android.sdk.internal.network.parsing.UriMoshiAdapter
|
||||
import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter
|
||||
import org.matrix.android.sdk.internal.session.sync.parsing.DefaultLazyRoomSyncJsonAdapter
|
||||
|
||||
object MoshiProvider {
|
||||
|
||||
|
|
@ -45,7 +45,8 @@ object MoshiProvider {
|
|||
.add(ForceToBooleanJsonAdapter())
|
||||
.add(CipherSuiteMoshiAdapter())
|
||||
.add(TlsVersionMoshiAdapter())
|
||||
.add(LazyRoomSyncJsonAdapter())
|
||||
// Use addLast here so we can inject a SplitLazyRoomSyncJsonAdapter later to override the default parsing.
|
||||
.addLast(DefaultLazyRoomSyncJsonAdapter())
|
||||
.add(RuntimeJsonAdapterFactory.of(MessageContent::class.java, "msgtype", MessageDefaultContent::class.java)
|
||||
.registerSubtype(MessageTextContent::class.java, MessageType.MSGTYPE_TEXT)
|
||||
.registerSubtype(MessageNoticeContent::class.java, MessageType.MSGTYPE_NOTICE)
|
||||
|
|
|
|||
|
|
@ -17,10 +17,7 @@
|
|||
package org.matrix.android.sdk.internal.session.sync
|
||||
|
||||
import okhttp3.ResponseBody
|
||||
import okio.buffer
|
||||
import okio.source
|
||||
import org.matrix.android.sdk.R
|
||||
import org.matrix.android.sdk.internal.di.MoshiProvider
|
||||
import org.matrix.android.sdk.internal.di.SessionFilesDirectory
|
||||
import org.matrix.android.sdk.internal.di.UserId
|
||||
import org.matrix.android.sdk.internal.network.GlobalErrorReceiver
|
||||
|
|
@ -32,8 +29,8 @@ import org.matrix.android.sdk.internal.session.filter.FilterRepository
|
|||
import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask
|
||||
import org.matrix.android.sdk.internal.session.reportSubtask
|
||||
import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync
|
||||
import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter
|
||||
import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
|
||||
import org.matrix.android.sdk.internal.session.sync.parsing.InitialSyncResponseParser
|
||||
import org.matrix.android.sdk.internal.session.user.UserStore
|
||||
import org.matrix.android.sdk.internal.task.Task
|
||||
import org.matrix.android.sdk.internal.util.logDuration
|
||||
|
|
@ -64,7 +61,8 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
private val syncTaskSequencer: SyncTaskSequencer,
|
||||
private val globalErrorReceiver: GlobalErrorReceiver,
|
||||
@SessionFilesDirectory
|
||||
private val fileDirectory: File
|
||||
private val fileDirectory: File,
|
||||
private val syncResponseParser: InitialSyncResponseParser
|
||||
) : SyncTask {
|
||||
|
||||
private val workingDir = File(fileDirectory, "is")
|
||||
|
|
@ -101,9 +99,10 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
val readTimeOut = (params.timeout + TIMEOUT_MARGIN).coerceAtLeast(TimeOutInterceptor.DEFAULT_LONG_TIMEOUT)
|
||||
|
||||
if (isInitialSync) {
|
||||
logDuration("INIT_SYNC strategy: $initialSyncStrategy") {
|
||||
if (initialSyncStrategy is InitialSyncStrategy.Optimized) {
|
||||
safeInitialSync(requestParams)
|
||||
val initSyncStrategy = initialSyncStrategy
|
||||
logDuration("INIT_SYNC strategy: $initSyncStrategy") {
|
||||
if (initSyncStrategy is InitialSyncStrategy.Optimized) {
|
||||
safeInitialSync(requestParams, initSyncStrategy)
|
||||
} else {
|
||||
val syncResponse = logDuration("INIT_SYNC Request") {
|
||||
executeRequest<SyncResponse>(globalErrorReceiver) {
|
||||
|
|
@ -132,7 +131,7 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}")
|
||||
}
|
||||
|
||||
private suspend fun safeInitialSync(requestParams: Map<String, String>) {
|
||||
private suspend fun safeInitialSync(requestParams: Map<String, String>, initSyncStrategy: InitialSyncStrategy.Optimized) {
|
||||
workingDir.mkdirs()
|
||||
val workingFile = File(workingDir, "initSync.json")
|
||||
val status = initialSyncStatusRepository.getStep()
|
||||
|
|
@ -163,7 +162,7 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
}
|
||||
initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED)
|
||||
}
|
||||
handleSyncFile(workingFile)
|
||||
handleSyncFile(workingFile, initSyncStrategy)
|
||||
|
||||
// Delete all files
|
||||
workingDir.deleteRecursively()
|
||||
|
|
@ -188,35 +187,12 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private suspend fun handleSyncFile(workingFile: File) {
|
||||
val syncResponseLength = workingFile.length().toInt()
|
||||
|
||||
logDuration("INIT_SYNC handleSyncFile() file size $syncResponseLength bytes") {
|
||||
if (syncResponseLength < (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToSplit ?: Long.MAX_VALUE) {
|
||||
// OK, no need to split just handle as a regular sync response
|
||||
Timber.v("INIT_SYNC no need to split")
|
||||
handleInitialSyncFile(workingFile)
|
||||
} else {
|
||||
Timber.v("INIT_SYNC Split into several smaller files")
|
||||
// Set file mode
|
||||
// TODO This is really ugly, I should improve that
|
||||
LazyRoomSyncJsonAdapter.initWith(workingFile)
|
||||
|
||||
handleInitialSyncFile(workingFile)
|
||||
|
||||
// Reset file mode
|
||||
LazyRoomSyncJsonAdapter.reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun handleInitialSyncFile(workingFile: File) {
|
||||
private suspend fun handleSyncFile(workingFile: File, initSyncStrategy: InitialSyncStrategy.Optimized) {
|
||||
logDuration("INIT_SYNC handleSyncFile()") {
|
||||
val syncResponse = logDuration("INIT_SYNC Read file and parse") {
|
||||
MoshiProvider.providesMoshi().adapter(SyncResponse::class.java)
|
||||
.fromJson(workingFile.source().buffer())!!
|
||||
syncResponseParser.parse(initSyncStrategy, workingFile)
|
||||
}
|
||||
initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED)
|
||||
|
||||
// Log some stats
|
||||
val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0
|
||||
val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored }
|
||||
|
|
@ -227,6 +203,7 @@ internal class DefaultSyncTask @Inject constructor(
|
|||
}
|
||||
initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val MAX_NUMBER_OF_RETRY_AFTER_TIMEOUT = 50
|
||||
|
|
|
|||
|
|
@ -16,17 +16,17 @@
|
|||
|
||||
package org.matrix.android.sdk.internal.session.sync.model
|
||||
|
||||
import com.squareup.moshi.JsonAdapter
|
||||
import com.squareup.moshi.JsonClass
|
||||
import com.squareup.moshi.JsonReader
|
||||
import okio.buffer
|
||||
import okio.source
|
||||
import org.matrix.android.sdk.internal.di.MoshiProvider
|
||||
import java.io.File
|
||||
|
||||
@JsonClass(generateAdapter = false)
|
||||
internal sealed class LazyRoomSync {
|
||||
data class Parsed(val _roomSync: RoomSync) : LazyRoomSync()
|
||||
data class Stored(val file: File) : LazyRoomSync()
|
||||
data class Stored(val roomSyncAdapter: JsonAdapter<RoomSync>, val file: File) : LazyRoomSync()
|
||||
|
||||
val roomSync: RoomSync
|
||||
get() {
|
||||
|
|
@ -35,8 +35,7 @@ internal sealed class LazyRoomSync {
|
|||
is Stored -> {
|
||||
// Parse the file now
|
||||
file.inputStream().use { pos ->
|
||||
MoshiProvider.providesMoshi().adapter(RoomSync::class.java)
|
||||
.fromJson(JsonReader.of(pos.source().buffer()))!!
|
||||
roomSyncAdapter.fromJson(JsonReader.of(pos.source().buffer()))!!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2021 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.session.sync.model
|
||||
|
||||
import com.squareup.moshi.FromJson
|
||||
import com.squareup.moshi.JsonAdapter
|
||||
import com.squareup.moshi.JsonReader
|
||||
import com.squareup.moshi.JsonWriter
|
||||
import com.squareup.moshi.ToJson
|
||||
import org.matrix.android.sdk.internal.di.MoshiProvider
|
||||
import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
|
||||
import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
internal class LazyRoomSyncJsonAdapter : JsonAdapter<LazyRoomSync>() {
|
||||
|
||||
@FromJson
|
||||
override fun fromJson(reader: JsonReader): LazyRoomSync {
|
||||
return if (workingDirectory != null) {
|
||||
val path = reader.path
|
||||
// val roomId = reader.path.substringAfter("\$.rooms.join.")
|
||||
|
||||
// inputStream.available() return 0... So read it to a String then decide to store in a file or to parse it now
|
||||
val json = reader.nextSource().inputStream().bufferedReader().use {
|
||||
it.readText()
|
||||
}
|
||||
|
||||
val limit = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToStoreInFile ?: Long.MAX_VALUE
|
||||
if (json.length > limit) {
|
||||
Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file")
|
||||
// Copy the source to a file
|
||||
val file = createFile()
|
||||
file.writeText(json)
|
||||
LazyRoomSync.Stored(file)
|
||||
} else {
|
||||
Timber.v("INIT_SYNC $path content length: ${json.length} parse it now")
|
||||
// Parse it now
|
||||
val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(json)!!
|
||||
LazyRoomSync.Parsed(roomSync)
|
||||
}
|
||||
} else {
|
||||
// Parse it now
|
||||
val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(reader)!!
|
||||
LazyRoomSync.Parsed(roomSync)
|
||||
}
|
||||
}
|
||||
|
||||
@ToJson
|
||||
override fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
|
||||
// This Adapter is not supposed to serialize object
|
||||
throw UnsupportedOperationException()
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun initWith(file: File) {
|
||||
workingDirectory = file.parentFile
|
||||
atomicInteger.set(0)
|
||||
}
|
||||
|
||||
fun reset() {
|
||||
workingDirectory = null
|
||||
}
|
||||
|
||||
private fun createFile(): File {
|
||||
val parent = workingDirectory ?: error("workingDirectory is not initialized")
|
||||
val index = atomicInteger.getAndIncrement()
|
||||
|
||||
return File(parent, "room_$index.json")
|
||||
}
|
||||
|
||||
private var workingDirectory: File? = null
|
||||
private val atomicInteger = AtomicInteger(0)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright (c) 2021 New Vector Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.session.sync.parsing
|
||||
|
||||
import com.squareup.moshi.Moshi
|
||||
import okio.buffer
|
||||
import okio.source
|
||||
import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
|
||||
import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import javax.inject.Inject
|
||||
|
||||
internal class InitialSyncResponseParser @Inject constructor(private val moshi: Moshi) {
|
||||
|
||||
fun parse(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File): SyncResponse {
|
||||
val syncResponseLength = workingFile.length().toInt()
|
||||
Timber.v("Sync file size is $syncResponseLength bytes")
|
||||
val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit
|
||||
Timber.v("INIT_SYNC should split in several files: $shouldSplit")
|
||||
return getMoshi(syncStrategy, workingFile, shouldSplit)
|
||||
.adapter(SyncResponse::class.java)
|
||||
.fromJson(workingFile.source().buffer())!!
|
||||
}
|
||||
|
||||
private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File, shouldSplit: Boolean): Moshi {
|
||||
// If we don't have to split we'll rely on the already default moshi
|
||||
if (!shouldSplit) return moshi
|
||||
// Otherwise, we create a new adapter for handling Map of Lazy sync
|
||||
return moshi.newBuilder()
|
||||
.add(SplitLazyRoomSyncJsonAdapter(workingFile, syncStrategy))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
/*
|
||||
* Copyright (c) 2021 New Vector Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.session.sync.parsing
|
||||
|
||||
import com.squareup.moshi.FromJson
|
||||
import com.squareup.moshi.JsonAdapter
|
||||
import com.squareup.moshi.JsonReader
|
||||
import com.squareup.moshi.JsonWriter
|
||||
import com.squareup.moshi.ToJson
|
||||
import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy
|
||||
import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync
|
||||
import org.matrix.android.sdk.internal.session.sync.model.RoomSync
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
internal class DefaultLazyRoomSyncJsonAdapter() {
|
||||
|
||||
@FromJson
|
||||
fun fromJson(reader: JsonReader, delegate: JsonAdapter<RoomSync>): LazyRoomSync? {
|
||||
val roomSync = delegate.fromJson(reader) ?: return null
|
||||
return LazyRoomSync.Parsed(roomSync)
|
||||
}
|
||||
|
||||
@ToJson
|
||||
fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
|
||||
// This Adapter is not supposed to serialize object
|
||||
Timber.v("To json $value with $writer")
|
||||
throw UnsupportedOperationException()
|
||||
}
|
||||
}
|
||||
|
||||
internal class SplitLazyRoomSyncJsonAdapter(
|
||||
private val workingDirectory: File,
|
||||
private val syncStrategy: InitialSyncStrategy.Optimized
|
||||
) {
|
||||
|
||||
private val atomicInteger = AtomicInteger(0)
|
||||
|
||||
private fun createFile(): File {
|
||||
val index = atomicInteger.getAndIncrement()
|
||||
return File(workingDirectory.parentFile, "room_$index.json")
|
||||
}
|
||||
|
||||
@FromJson
|
||||
fun fromJson(reader: JsonReader, delegate: JsonAdapter<RoomSync>): LazyRoomSync? {
|
||||
val path = reader.path
|
||||
val json = reader.nextSource().inputStream().bufferedReader().use {
|
||||
it.readText()
|
||||
}
|
||||
val limit = syncStrategy.minSizeToStoreInFile
|
||||
return if (json.length > limit) {
|
||||
Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file")
|
||||
// Copy the source to a file
|
||||
val file = createFile()
|
||||
file.writeText(json)
|
||||
LazyRoomSync.Stored(delegate, file)
|
||||
} else {
|
||||
Timber.v("INIT_SYNC $path content length: ${json.length} parse it now")
|
||||
val roomSync = delegate.fromJson(json) ?: return null
|
||||
LazyRoomSync.Parsed(roomSync)
|
||||
}
|
||||
}
|
||||
|
||||
@ToJson
|
||||
fun toJson(writer: JsonWriter, value: LazyRoomSync?) {
|
||||
// This Adapter is not supposed to serialize object
|
||||
Timber.v("To json $value with $writer")
|
||||
throw UnsupportedOperationException()
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue