mirrors/nextcloud-android
1
0
Fork 0
mirror of https://github.com/nextcloud/android.git synced 2024-11-26 07:05:49 +03:00
Code Issues Projects Releases Packages Wiki Activity

draft implement faster auto upload file changed scan

Browse source 
Signed-off-by: Jonas Mayer <jonas.a.mayer@gmx.net>
This commit is contained in:
Jonas Mayer 2024-04-03 18:04:54 +02:00 committed by Jonas Mayer
parent 2a816c7d19
commit 8f3b5b106c
8 changed files with 125 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/src/androidTest/java/com/owncloud/android/utils/SyncedFolderUtilsTest.kt
View file

@ -174,7 +174,8 @@ class SyncedFolderUtilsTest : AbstractIT() {
MediaFolderType.IMAGE,
false,
SubFolderRule.YEAR_MONTH,
false
false,
SyncedFolder.NOT_SCANNED_YET
)
Assert.assertFalse(SyncedFolderUtils.isQualifyingMediaFolder(folder))
}
@ -198,7 +199,8 @@ class SyncedFolderUtilsTest : AbstractIT() {
MediaFolderType.IMAGE,
false,
SubFolderRule.YEAR_MONTH,
false
false,
SyncedFolder.NOT_SCANNED_YET
)
Assert.assertFalse(SyncedFolderUtils.isQualifyingMediaFolder(folder))
}

6
app/src/main/java/com/nextcloud/client/database/entity/SyncedFolderEntity.kt
View file

@ -45,6 +45,8 @@ data class SyncedFolderEntity(
val hidden: Int?,
@ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE)
val subFolderRule: Int?,
@ColumnInfo(name = ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN)
val excludeHidden: Int?
@ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN)
val excludeHidden: Int?,
@ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS)
val lastScanTimestampMs: Long?
)

16
app/src/main/java/com/owncloud/android/datamodel/SyncedFolder.java
View file

@ -21,6 +21,7 @@ import java.io.Serializable;
public class SyncedFolder implements Serializable, Cloneable {
public static final long UNPERSISTED_ID = Long.MIN_VALUE;
public static final long EMPTY_ENABLED_TIMESTAMP_MS = -1;
public static final long NOT_SCANNED_YET = -1;
private static final long serialVersionUID = -793476118299906429L;
@ -41,6 +42,7 @@ public class SyncedFolder implements Serializable, Cloneable {
private boolean hidden;
private SubFolderRule subfolderRule;
private boolean excludeHidden;
private long lastScanTimestampMs;
/**
* constructor for new, to be persisted entity.
@ -75,7 +77,8 @@ public class SyncedFolder implements Serializable, Cloneable {
MediaFolderType type,
boolean hidden,
SubFolderRule subFolderRule,
boolean excludeHidden) {
boolean excludeHidden,
long lastScanTimestampMs) {
this(UNPERSISTED_ID,
localPath,
remotePath,
@ -91,7 +94,8 @@ public class SyncedFolder implements Serializable, Cloneable {
type,
hidden,
subFolderRule,
excludeHidden);
excludeHidden,
lastScanTimestampMs);
}
/**
@ -114,7 +118,8 @@ public class SyncedFolder implements Serializable, Cloneable {
MediaFolderType type,
boolean hidden,
SubFolderRule subFolderRule,
boolean excludeHidden) {
boolean excludeHidden,
long lastScanTimestampMs) {
this.id = id;
this.localPath = localPath;
this.remotePath = remotePath;
@ -130,6 +135,7 @@ public class SyncedFolder implements Serializable, Cloneable {
this.hidden = hidden;
this.subfolderRule = subFolderRule;
this.excludeHidden = excludeHidden;
this.lastScanTimestampMs = lastScanTimestampMs;
}
/**
@ -271,4 +277,8 @@ public class SyncedFolder implements Serializable, Cloneable {
public boolean containsFile(String filePath){
return filePath.contains(localPath);
}
public long getLastScanTimestampMs() { return lastScanTimestampMs; }
public void setLastScanTimestampMs(long lastScanTimestampMs) { this.lastScanTimestampMs = lastScanTimestampMs; }
}

12
app/src/main/java/com/owncloud/android/datamodel/SyncedFolderDisplayItem.java
View file

@ -60,7 +60,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
MediaFolderType type,
boolean hidden,
SubFolderRule subFolderRule,
boolean excludeHidden) {
boolean excludeHidden,
long lastScanTimestampMs) {
super(id,
localPath,
remotePath,
@ -76,7 +77,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
type,
hidden,
subFolderRule,
excludeHidden);
excludeHidden,
lastScanTimestampMs);
this.filePaths = filePaths;
this.folderName = folderName;
this.numberOfFiles = numberOfFiles;
@ -98,7 +100,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
MediaFolderType type,
boolean hidden,
SubFolderRule subFolderRule,
boolean excludeHidden) {
boolean excludeHidden,
long lastScanTimestampMs) {
super(id,
localPath,
remotePath,
@ -114,7 +117,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
type,
hidden,
subFolderRule,
excludeHidden);
excludeHidden,
lastScanTimestampMs);
this.folderName = folderName;
}

11
app/src/main/java/com/owncloud/android/datamodel/SyncedFolderProvider.java
View file

@ -360,7 +360,9 @@ public class SyncedFolderProvider extends Observable {
SubFolderRule subFolderRule = SubFolderRule.values()[cursor.getInt(
cursor.getColumnIndexOrThrow(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE))];
boolean excludeHidden = cursor.getInt(cursor.getColumnIndexOrThrow(
ProviderMeta.ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN)) == 1;
ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN)) == 1;
long lastScanTimestampMs = cursor.getLong(cursor.getColumnIndexOrThrow(
ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS));
syncedFolder = new SyncedFolder(id,
@ -378,7 +380,8 @@ public class SyncedFolderProvider extends Observable {
type,
hidden,
subFolderRule,
excludeHidden);
excludeHidden,
lastScanTimestampMs);
}
return syncedFolder;
}
@ -407,8 +410,8 @@ public class SyncedFolderProvider extends Observable {
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_TYPE, syncedFolder.getType().id);
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_HIDDEN, syncedFolder.isHidden());
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE, syncedFolder.getSubfolderRule().ordinal());
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN, syncedFolder.isExcludeHidden());
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN, syncedFolder.isExcludeHidden());
cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS, syncedFolder.getLastScanTimestampMs());
return cv;
}

3
app/src/main/java/com/owncloud/android/db/ProviderMeta.java
View file

@ -293,7 +293,8 @@ public class ProviderMeta {
public static final String SYNCED_FOLDER_NAME_COLLISION_POLICY = "name_collision_policy";
public static final String SYNCED_FOLDER_HIDDEN = "hidden";
public static final String SYNCED_FOLDER_SUBFOLDER_RULE = "sub_folder_rule";
public static final String SYNCED_EXCLUDE_HIDDEN = "exclude_hidden";
public static final String SYNCED_FOLDER_EXCLUDE_HIDDEN = "exclude_hidden";
public static final String SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS = "last_scan_timestamp_ms";
// Columns of external links table
public static final String EXTERNAL_LINKS_ICON_URL = "icon_url";

15
app/src/main/java/com/owncloud/android/ui/activity/SyncedFoldersActivity.kt
View file

@ -387,7 +387,8 @@ class SyncedFoldersActivity :
syncedFolder.type,
syncedFolder.isHidden,
syncedFolder.subfolderRule,
syncedFolder.isExcludeHidden
syncedFolder.isExcludeHidden,
syncedFolder.lastScanTimestampMs
)
}
@ -418,7 +419,8 @@ class SyncedFoldersActivity :
mediaFolder.type,
syncedFolder.isHidden,
syncedFolder.subfolderRule,
syncedFolder.isExcludeHidden
syncedFolder.isExcludeHidden,
syncedFolder.lastScanTimestampMs
)
}
@ -448,7 +450,8 @@ class SyncedFoldersActivity :
mediaFolder.type,
false,
SubFolderRule.YEAR_MONTH,
false
false,
SyncedFolder.NOT_SCANNED_YET
)
}
@ -541,7 +544,8 @@ class SyncedFoldersActivity :
MediaFolderType.CUSTOM,
false,
SubFolderRule.YEAR_MONTH,
false
false,
SyncedFolder.NOT_SCANNED_YET
)
onSyncFolderSettingsClick(0, emptyCustomFolder)
} else {
@ -658,7 +662,8 @@ class SyncedFoldersActivity :
syncedFolder.type,
syncedFolder.isHidden,
syncedFolder.subFolderRule,
syncedFolder.isExcludeHidden
syncedFolder.isExcludeHidden,
SyncedFolder.NOT_SCANNED_YET
)
saveOrUpdateSyncedFolder(newCustomFolder)
adapter.addSyncFolderItem(newCustomFolder)

154
app/src/main/java/com/owncloud/android/utils/FilesSyncHelper.java
View file

@ -55,6 +55,59 @@ public final class FilesSyncHelper {
// utility class -> private constructor
}
private static void insertCustomFolderIntoDB(Path path,
SyncedFolder syncedFolder,
FilesystemDataProvider filesystemDataProvider,
long lastCheck,
long thisCheck) {
final long enabledTimestampMs = syncedFolder.getEnabledTimestampMs();
try {
FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
File file = path.toFile();
if (syncedFolder.isExcludeHidden() && file.isHidden()) {
// exclude hidden file or folder
return FileVisitResult.CONTINUE;
}
if (lastCheck != SyncedFolder.NOT_SCANNED_YET && attrs.lastModifiedTime().toMillis() < lastCheck) {
// skip files that were already checked
return FileVisitResult.CONTINUE;
}
if (syncedFolder.isExisting() || attrs.lastModifiedTime().toMillis() >= enabledTimestampMs) {
// storeOrUpdateFileValue takes a few ms
// -> Rest of this file check takes not even 1 ms.
filesystemDataProvider.storeOrUpdateFileValue(path.toAbsolutePath().toString(),
attrs.lastModifiedTime().toMillis(),
file.isDirectory(), syncedFolder);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
return null;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return FileVisitResult.CONTINUE;
}
});
syncedFolder.setLastScanTimestampMs(thisCheck);
} catch (IOException e) {
Log_OC.e(TAG, "Something went wrong while indexing files for auto upload", e);
}
}
private static void insertAllDBEntriesForSyncedFolder(SyncedFolder syncedFolder) {
final Context context = MainApp.getAppContext();
final ContentResolver contentResolver = context.getContentResolver();
@ -63,92 +116,31 @@ public final class FilesSyncHelper {
if (syncedFolder.isEnabled() && (syncedFolder.isExisting() || enabledTimestampMs >= 0)) {
MediaFolderType mediaType = syncedFolder.getType();
final long lastCheck = syncedFolder.getLastScanTimestampMs();
final long thisCheck = System.currentTimeMillis();
if (mediaType == MediaFolderType.IMAGE) {
FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.INTERNAL_CONTENT_URI
, syncedFolder);
FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.INTERNAL_CONTENT_URI,
syncedFolder,
lastCheck, thisCheck);
FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
syncedFolder);
syncedFolder,
lastCheck, thisCheck);
} else if (mediaType == MediaFolderType.VIDEO) {
FilesSyncHelper.insertContentIntoDB(MediaStore.Video.Media.INTERNAL_CONTENT_URI,
syncedFolder);
syncedFolder,
lastCheck, thisCheck);
FilesSyncHelper.insertContentIntoDB(MediaStore.Video.Media.EXTERNAL_CONTENT_URI,
syncedFolder);
syncedFolder,
lastCheck, thisCheck);
} else {
try {
FilesystemDataProvider filesystemDataProvider = new FilesystemDataProvider(contentResolver);
Path path = Paths.get(syncedFolder.getLocalPath());
long startTime = System.nanoTime();
// chick check for changes
long lastCheck = System.currentTimeMillis();
ArrayList<File> changedFiles = new ArrayList<>();
FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
File file = path.toFile();
if (syncedFolder.isExcludeHidden() && file.isHidden()) {
// exclude hidden file or folder
return FileVisitResult.CONTINUE;
}
if (file.lastModified() >= lastCheck){
changedFiles.add(file);
}
FilesSyncHelper.insertCustomFolderIntoDB(path, syncedFolder, filesystemDataProvider, lastCheck, thisCheck);
Log_OC.d(TAG,"FILESYNC FINISHED LONG CHECK FOLDER "+path+" "+(System.nanoTime() - startTime));
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
return null;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return FileVisitResult.CONTINUE;
}
});
Log_OC.d(TAG,"FILESYNC FINISHED QUICK CHECK FILE "+path+" "+(System.nanoTime() - startTime));
startTime = System.nanoTime();
FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
File file = path.toFile();
if (syncedFolder.isExcludeHidden() && file.isHidden()) {
// exclude hidden file or folder
return FileVisitResult.CONTINUE;
}
if (syncedFolder.isExisting() || attrs.lastModifiedTime().toMillis() >= enabledTimestampMs) {
// storeOrUpdateFileValue takes a few millisec
// -> Rest of this file check takes not even 1 millisec.
filesystemDataProvider.storeOrUpdateFileValue(path.toAbsolutePath().toString(),
attrs.lastModifiedTime().toMillis(),
file.isDirectory(), syncedFolder);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
return null;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return FileVisitResult.CONTINUE;
}
});
Log_OC.d(TAG,"FILESYNC FINISHED LONG CHECK FILE "+path+" "+(System.nanoTime() - startTime));
} catch (IOException e) {
Log_OC.e(TAG, "Something went wrong while indexing files for auto upload", e);
}
}
}
}
@ -200,7 +192,7 @@ public final class FilesSyncHelper {
return filePath;
}
private static void insertContentIntoDB(Uri uri, SyncedFolder syncedFolder) {
private static void insertContentIntoDB(Uri uri, SyncedFolder syncedFolder, long lastCheckMs, long thisCheckMs) {
final Context context = MainApp.getAppContext();
final ContentResolver contentResolver = context.getContentResolver();
@ -231,15 +223,23 @@ public final class FilesSyncHelper {
column_index_date_modified = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DATE_MODIFIED);
while (cursor.moveToNext()) {
contentPath = cursor.getString(column_index_data);
Log_OC.d(TAG,"FILESYNC CHECK File "+contentPath);
isFolder = new File(contentPath).isDirectory();
if (syncedFolder.getLastScanTimestampMs() != SyncedFolder.NOT_SCANNED_YET &&
cursor.getLong(column_index_date_modified) < (lastCheckMs / 1000.0)) {
continue;
}
if (syncedFolder.isExisting() || cursor.getLong(column_index_date_modified) >= enabledTimestampMs / 1000.0) {
// storeOrUpdateFileValue takes a few ms
// -> Rest of this file check takes not even 1 ms.
filesystemDataProvider.storeOrUpdateFileValue(contentPath,
cursor.getLong(column_index_date_modified), isFolder,
syncedFolder);
}
}
cursor.close();
syncedFolder.setLastScanTimestampMs(thisCheckMs);
}
}