mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-10-27 06:45:49 +03:00
Code Issues Projects Releases Packages Wiki Activity

New Discovery Algo: Refactor by splitting the processFile in two

Browse source
This commit is contained in:
Olivier Goffart 2018-10-05 11:05:08 +02:00 committed by Kevin Ottens
parent b7d827d6d0
commit fc69dda246
No known key found for this signature in database
GPG key ID: 074BBBCB8DECC9E2
2 changed files with 269 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
src/libsync/discovery.cpp
View file

@ -383,27 +383,41 @@ void ProcessDirectoryJob::processFile(PathTuple path,
item->_file = path._target;
item->_originalFile = path._original;
auto computeLocalChecksum = [&](const QByteArray &type, const QString &path) {
if (_queryServer == NormalQuery && serverEntry.isValid()) {
processFileAnalyzeRemoteInfo(item, path, localEntry, serverEntry, dbEntry);
return;
} else if (_queryServer == ParentNotChanged && dbEntry._type == ItemTypeVirtualFileDownload) {
// download virtual file
item->_direction = SyncFileItem::Down;
item->_instruction = CSYNC_INSTRUCTION_NEW;
Q_ASSERT(item->_file.endsWith(_discoveryData->_syncOptions._virtualFileSuffix));
item->_file.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
item->_type = ItemTypeVirtualFileDownload;
}
processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer);
}
// Compute the checksum of the given file and assign the result in item->_checksumHeader
// Returns true if the checksum was successfully computed
static bool computeLocalChecksum(const QByteArray &header, const QString &path, const SyncFileItemPtr &item)
{
auto type = parseChecksumHeaderType(header);
if (!type.isEmpty()) {
// TODO: compute async?
QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path, type);
QByteArray checksum = ComputeChecksum::computeNow(path, type);
if (!checksum.isEmpty()) {
item->_checksumHeader = makeChecksumHeader(type, checksum);
return true;
}
}
return false;
};
}
bool noServerEntry = (_queryServer != ParentNotChanged && !serverEntry.isValid())
|| (_queryServer == ParentNotChanged && !dbEntry.isValid());
auto recurseQueryServer = _queryServer;
if (noServerEntry)
recurseQueryServer = ParentDontExist;
if (_queryServer == NormalQuery && serverEntry.isValid()) {
void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
const SyncFileItemPtr &item, PathTuple path, const LocalInfo &localEntry,
const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry)
{
item->_checksumHeader = serverEntry.checksumHeader;
item->_fileId = serverEntry.fileId;
item->_remotePerm = serverEntry.remotePerm;
@ -435,6 +449,8 @@ void ProcessDirectoryJob::processFile(PathTuple path,
if (!localEntry.isValid()) {
// Check for renames (if there is a file with the same file id)
bool done = false;
bool async = false;
// This function will be executed for every candidate
auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) {
if (done)
return;
@ -561,7 +577,7 @@ void ProcessDirectoryJob::processFile(PathTuple path,
qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->isDirectory();
if (item->isDirectory()) {
auto job = new ProcessDirectoryJob(item, recurseQueryServer,
auto job = new ProcessDirectoryJob(item, _queryServer,
item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
_discoveryData, this);
job->_currentFolder = path;
@ -575,14 +591,14 @@ void ProcessDirectoryJob::processFile(PathTuple path,
});
job->start();
done = true; // Ideally, if the origin still exist on the server, we should continue searching... but that'd be difficult
item = nullptr;
async = true;
}
};
if (!_discoveryData->_statedb->getFileRecordsByFileId(serverEntry.fileId, renameCandidateProcessing)) {
dbError();
return;
}
if (!item) {
if (async) {
return; // We went async
}
}
@ -618,18 +634,26 @@ void ProcessDirectoryJob::processFile(PathTuple path,
item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
item->_direction = SyncFileItem::Down;
} else {
recurseQueryServer = ParentNotChanged;
processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, ParentNotChanged);
return;
}
} else if (_queryServer == ParentNotChanged && dbEntry._type == ItemTypeVirtualFileDownload) {
item->_direction = SyncFileItem::Down;
item->_instruction = CSYNC_INSTRUCTION_NEW;
Q_ASSERT(item->_file.endsWith(_discoveryData->_syncOptions._virtualFileSuffix));
item->_file.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
item->_type = ItemTypeVirtualFileDownload;
processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer);
}
void ProcessDirectoryJob::processFileAnalyzeLocalInfo(
const SyncFileItemPtr &item, PathTuple path, const LocalInfo &localEntry,
const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry, QueryMode recurseQueryServer)
{
bool noServerEntry = (_queryServer != ParentNotChanged && !serverEntry.isValid())
|| (_queryServer == ParentNotChanged && !dbEntry.isValid());
if (noServerEntry)
recurseQueryServer = ParentDontExist;
bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC
|| item->_instruction == CSYNC_INSTRUCTION_RENAME || item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE;
if ((dbEntry.isValid() && dbEntry._type == ItemTypeVirtualFile) || (localEntry.isValid() && localEntry.isVirtualFile && item->_type != ItemTypeVirtualFileDownload)) {
// Do not download virtual files
if (serverModified || dbEntry._type != ItemTypeVirtualFile)
@ -670,9 +694,7 @@ void ProcessDirectoryJob::processFile(PathTuple path,
ASSERT(item->_type == ItemTypeVirtualFile)
ASSERT(item->_file.endsWith(_discoveryData->_syncOptions._virtualFileSuffix));
item->_file.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
} else if (dbEntry.isValid() && !typeChange &&
((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size)
|| (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) {
} else if (dbEntry.isValid() && !typeChange && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) {
// Local file unchanged.
if (noServerEntry) {
item->_instruction = CSYNC_INSTRUCTION_REMOVE;
@ -804,7 +826,7 @@ void ProcessDirectoryJob::processFile(PathTuple path,
// Verify the checksum where possible
if (isMove && !base._checksumHeader.isEmpty() && item->_type == ItemTypeFile) {
if (computeLocalChecksum(parseChecksumHeaderType(base._checksumHeader), path._original)) {
if (computeLocalChecksum(base._checksumHeader, _discoveryData->_localDir + path._original, item)) {
qCInfo(lcDisco) << "checking checksum of potential rename " << path._original << item->_checksumHeader << base._checksumHeader;
isMove = item->_checksumHeader == base._checksumHeader;
}
@ -955,7 +977,8 @@ void ProcessDirectoryJob::processFile(PathTuple path,
// check #4754 #4755
bool isEmlFile = path._original.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive);
if (isEmlFile && dbEntry._fileSize == localEntry.size && !dbEntry._checksumHeader.isEmpty()) {
if (computeLocalChecksum(parseChecksumHeaderType(dbEntry._checksumHeader), path._local) && item->_checksumHeader == dbEntry._checksumHeader) {
if (computeLocalChecksum(dbEntry._checksumHeader, _discoveryData->_localDir + path._local, item)
&& item->_checksumHeader == dbEntry._checksumHeader) {
qCInfo(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path._local;
item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
}

6
src/libsync/discovery.h
View file

@ -73,7 +73,11 @@ private:
void process();
// return true if the file is excluded
bool handleExcluded(const QString &path, bool isDirectory, bool isHidden, bool isSymlink);
void processFile(PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &dbEntry);
void processFile(PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &);
void processFileAnalyzeRemoteInfo(const SyncFileItemPtr &item, PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &);
void processFileAnalyzeLocalInfo(const SyncFileItemPtr &item, PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &, QueryMode recurseQueryServer);
// Return false if there is an error and that a directory must not be recursively be taken
bool checkPermissions(const SyncFileItemPtr &item);
void processBlacklisted(const PathTuple &, const LocalInfo &, const SyncJournalFileRecord &dbEntry);