From b977b44295ae82222bc9907a33b2df7ba38fb5e6 Mon Sep 17 00:00:00 2001 From: alperozturk Date: Wed, 11 Dec 2024 13:26:41 +0100 Subject: [PATCH 1/2] get-files-for-upload Signed-off-by: alperozturk --- .../nextcloud/client/jobs/FilesSyncWork.kt | 37 ++++++++++++-- .../datamodel/FilesystemDataProvider.java | 48 ------------------- 2 files changed, 32 insertions(+), 53 deletions(-) diff --git a/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt b/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt index b1d3fe8002..4d9de76e35 100644 --- a/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt +++ b/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt @@ -37,6 +37,8 @@ import com.owncloud.android.utils.FileStorageUtils import com.owncloud.android.utils.FilesSyncHelper import com.owncloud.android.utils.MimeType import com.owncloud.android.utils.MimeTypeUtil +import com.owncloud.android.utils.SyncedFolderUtils.isFileNameQualifiedForAutoUpload +import com.owncloud.android.utils.SyncedFolderUtils.isQualifiedFolder import java.io.File import java.text.ParsePosition import java.text.SimpleDateFormat @@ -217,6 +219,30 @@ class FilesSyncWork( syncedFolderProvider.updateSyncFolder(syncedFolder) } + private fun getAllFiles(path: String): Set { + val result = mutableListOf() + val directory = File(path) + + if (directory.exists()) { + val files = directory.listFiles() + + files?.forEach { file -> + if (file.isFile) { + if (file.exists() && + isQualifiedFolder(file.getParent()) && + isFileNameQualifiedForAutoUpload(file.getName())) { + result.add(file.path) + } + } else if (file.isDirectory) { + val allFiles = getAllFiles(file.path) + result.addAll(allFiles) + } + } + } + + return result.toSet() + } + @Suppress("LongMethod") // legacy code private fun uploadFilesFromFolder( context: Context, @@ -232,22 +258,21 @@ class FilesSyncWork( val needsWifi: Boolean var file: File val accountName = syncedFolder.account + val optionalUser = userAccountManager.getUser(accountName) if (!optionalUser.isPresent) { return } + val user = optionalUser.get() val arbitraryDataProvider: ArbitraryDataProvider? = if (lightVersion) { ArbitraryDataProviderImpl(context) } else { null } - val paths = filesystemDataProvider.getFilesForUpload( - syncedFolder.localPath, - syncedFolder.id.toString() - ) - if (paths.size == 0) { + val paths = getAllFiles(syncedFolder.localPath) + if (paths.isEmpty()) { return } @@ -260,6 +285,7 @@ class FilesSyncWork( MimeTypeUtil.getBestMimeTypeByFilename(localPath) ) } + val localPaths = pathsAndMimes.map { it.first }.toTypedArray() val remotePaths = pathsAndMimes.map { it.second }.toTypedArray() @@ -276,6 +302,7 @@ class FilesSyncWork( needsWifi = syncedFolder.isWifiOnly uploadAction = syncedFolder.uploadAction } + FileUploadHelper.instance().uploadNewFiles( user, localPaths, diff --git a/app/src/main/java/com/owncloud/android/datamodel/FilesystemDataProvider.java b/app/src/main/java/com/owncloud/android/datamodel/FilesystemDataProvider.java index f4b6a03ef2..ed22265325 100644 --- a/app/src/main/java/com/owncloud/android/datamodel/FilesystemDataProvider.java +++ b/app/src/main/java/com/owncloud/android/datamodel/FilesystemDataProvider.java @@ -14,16 +14,11 @@ import android.net.Uri; import com.owncloud.android.db.ProviderMeta; import com.owncloud.android.lib.common.utils.Log_OC; -import com.owncloud.android.utils.SyncedFolderUtils; import java.io.BufferedInputStream; -import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.HashSet; -import java.util.Set; import java.util.zip.CRC32; /** @@ -63,49 +58,6 @@ public class FilesystemDataProvider { ); } - public Set getFilesForUpload(String localPath, String syncedFolderId) { - Set localPathsToUpload = new HashSet<>(); - - String likeParam = localPath + "%"; - - Cursor cursor = contentResolver.query( - ProviderMeta.ProviderTableMeta.CONTENT_URI_FILESYSTEM, - null, - ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_LOCAL_PATH + " LIKE ? and " + - ProviderMeta.ProviderTableMeta.FILESYSTEM_SYNCED_FOLDER_ID + " = ? and " + - ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_SENT_FOR_UPLOAD + " = ? and " + - ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_IS_FOLDER + " = ?", - new String[]{likeParam, syncedFolderId, "0", "0"}, - null); - - if (cursor != null) { - if (cursor.moveToFirst()) { - do { - String value = cursor.getString(cursor.getColumnIndexOrThrow( - ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_LOCAL_PATH)); - if (value == null) { - Log_OC.e(TAG, "Cannot get local path"); - } else { - File file = new File(value); - if (!file.exists()) { - Log_OC.d(TAG, "Ignoring file for upload (doesn't exist): " + value); - } else if (!SyncedFolderUtils.isQualifiedFolder(file.getParent())) { - Log_OC.d(TAG, "Ignoring file for upload (unqualified folder): " + value); - } else if (!SyncedFolderUtils.isFileNameQualifiedForAutoUpload(file.getName())) { - Log_OC.d(TAG, "Ignoring file for upload (unqualified file): " + value); - } else { - localPathsToUpload.add(value); - } - } - } while (cursor.moveToNext()); - } - - cursor.close(); - } - - return localPathsToUpload; - } - public void storeOrUpdateFileValue(String localPath, long modifiedAt, boolean isFolder, SyncedFolder syncedFolder) { // takes multiple milliseconds to query data from database (around 75% of execution time) (6ms) From 66cf7f83eb0723b8091c76c70cb2cc7fb504c306 Mon Sep 17 00:00:00 2001 From: alperozturk Date: Wed, 11 Dec 2024 13:50:52 +0100 Subject: [PATCH 2/2] return files directly Signed-off-by: alperozturk --- .../nextcloud/client/jobs/FilesSyncWork.kt | 53 +++++++------------ 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt b/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt index 4d9de76e35..40464964d3 100644 --- a/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt +++ b/app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt @@ -23,7 +23,6 @@ import com.nextcloud.client.jobs.upload.FileUploadWorker import com.nextcloud.client.network.ConnectivityService import com.nextcloud.client.preferences.SubFolderRule import com.owncloud.android.R -import com.owncloud.android.datamodel.ArbitraryDataProvider import com.owncloud.android.datamodel.ArbitraryDataProviderImpl import com.owncloud.android.datamodel.FilesystemDataProvider import com.owncloud.android.datamodel.MediaFolderType @@ -219,28 +218,18 @@ class FilesSyncWork( syncedFolderProvider.updateSyncFolder(syncedFolder) } - private fun getAllFiles(path: String): Set { - val result = mutableListOf() - val directory = File(path) - - if (directory.exists()) { - val files = directory.listFiles() - - files?.forEach { file -> - if (file.isFile) { - if (file.exists() && - isQualifiedFolder(file.getParent()) && - isFileNameQualifiedForAutoUpload(file.getName())) { - result.add(file.path) - } - } else if (file.isDirectory) { - val allFiles = getAllFiles(file.path) - result.addAll(allFiles) - } + private fun getAllFiles(path: String): Set { + return File(path).takeIf { it.exists() } + ?.walkTopDown() + ?.asSequence() + ?.filter { file -> + file.isFile && + file.exists() && + isQualifiedFolder(file.parentFile?.path) && + isFileNameQualifiedForAutoUpload(file.name) } - } - - return result.toSet() + ?.toSet() + ?: emptySet() } @Suppress("LongMethod") // legacy code @@ -256,7 +245,6 @@ class FilesSyncWork( val uploadAction: Int? val needsCharging: Boolean val needsWifi: Boolean - var file: File val accountName = syncedFolder.account val optionalUser = userAccountManager.getUser(accountName) @@ -265,19 +253,18 @@ class FilesSyncWork( } val user = optionalUser.get() - val arbitraryDataProvider: ArbitraryDataProvider? = if (lightVersion) { + val arbitraryDataProvider = if (lightVersion) { ArbitraryDataProviderImpl(context) } else { null } - val paths = getAllFiles(syncedFolder.localPath) - if (paths.isEmpty()) { + val files = getAllFiles(syncedFolder.localPath) + if (files.isEmpty()) { return } - val pathsAndMimes = paths.map { path -> - file = File(path) + val pathsAndMimes = files.map { file -> val localPath = file.absolutePath Triple( localPath, @@ -291,10 +278,11 @@ class FilesSyncWork( if (lightVersion) { needsCharging = resources.getBoolean(R.bool.syncedFolder_light_on_charging) - needsWifi = arbitraryDataProvider!!.getBooleanValue( + needsWifi = arbitraryDataProvider?.getBooleanValue( accountName, SettingsActivity.SYNCED_FOLDER_LIGHT_UPLOAD_ON_WIFI - ) + ) ?: true + val uploadActionString = resources.getString(R.string.syncedFolder_light_upload_behaviour) uploadAction = getUploadAction(uploadActionString) } else { @@ -316,10 +304,9 @@ class FilesSyncWork( syncedFolder.nameCollisionPolicy ) - for (path in paths) { - // TODO batch update + for (file in files) { filesystemDataProvider.updateFilesystemFileAsSentForUpload( - path, + file.path, syncedFolder.id.toString() ) }