From ee211d76093eeb19e556e01c6917d58025d53f97 Mon Sep 17 00:00:00 2001 From: Jocelyn Turcotte Date: Wed, 25 Jan 2017 11:28:18 +0100 Subject: [PATCH] Release SyncFileItem objects with their job We now delete subjobs as their propagation is complete. This allows us to also release the item by making sure that nothing else is holding a reference to it. Remove the stored SyncFileItemVector from SyncEngine and SyncResult and instead gather the needed info progressively as each itemCompleted signal is emitted. This frees some holes on the heap as propagation goes, allowing many memory allocations without the need of requesting more virtual memory from the OS, preventing the memory usage from increasingly growing. --- src/gui/folder.cpp | 179 +++++++--------------------------- src/gui/folder.h | 6 +- src/gui/folderman.cpp | 6 +- src/gui/folderman.h | 2 +- src/gui/folderstatusmodel.cpp | 16 +-- src/libsync/syncengine.cpp | 62 ++++++------ src/libsync/syncengine.h | 17 +--- src/libsync/syncresult.cpp | 111 ++++++++++++++------- src/libsync/syncresult.h | 50 +++++++--- 9 files changed, 197 insertions(+), 252 deletions(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 9d805be02..6aa0fe2fb 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -52,7 +52,6 @@ Folder::Folder(const FolderDefinition& definition, : QObject(parent) , _accountState(accountState) , _definition(definition) - , _csyncError(false) , _csyncUnavail(false) , _wipeDb(false) , _proxyDirty(true) @@ -87,8 +86,6 @@ Folder::Folder(const FolderDefinition& definition, connect(_accountState.data(), SIGNAL(isConnectedChanged()), this, SIGNAL(canSyncChanged())); connect(_engine.data(), SIGNAL(rootEtag(QString)), this, SLOT(etagRetreivedFromSyncEngine(QString))); - connect(_engine.data(), SIGNAL(treeWalkResult(const SyncFileItemVector&)), - this, SLOT(slotThreadTreeWalkResult(const SyncFileItemVector&)), Qt::QueuedConnection); connect(_engine.data(), SIGNAL(started()), SLOT(slotSyncStarted()), Qt::QueuedConnection); connect(_engine.data(), SIGNAL(finished(bool)), SLOT(slotSyncFinished(bool)), Qt::QueuedConnection); @@ -138,13 +135,13 @@ void Folder::checkLocalPath() } else { // Check directory again if( !FileSystem::fileExists(_definition.localPath, fi) ) { - _syncResult.setErrorString(tr("Local folder %1 does not exist.").arg(_definition.localPath)); + _syncResult.appendErrorString(tr("Local folder %1 does not exist.").arg(_definition.localPath)); _syncResult.setStatus( SyncResult::SetupError ); } else if( !fi.isDir() ) { - _syncResult.setErrorString(tr("%1 should be a folder but is not.").arg(_definition.localPath)); + _syncResult.appendErrorString(tr("%1 should be a folder but is not.").arg(_definition.localPath)); _syncResult.setStatus( SyncResult::SetupError ); } else if( !fi.isReadable() ) { - _syncResult.setErrorString(tr("%1 is not readable.").arg(_definition.localPath)); + _syncResult.appendErrorString(tr("%1 is not readable.").arg(_definition.localPath)); _syncResult.setStatus( SyncResult::SetupError ); } } @@ -267,8 +264,8 @@ SyncResult Folder::syncResult() const void Folder::prepareToSync() { + _syncResult.reset(); _syncResult.setStatus( SyncResult::NotYetStarted ); - _syncResult.clearErrors(); } void Folder::slotRunEtagJob() @@ -322,120 +319,33 @@ void Folder::etagRetreivedFromSyncEngine(const QString& etag) } -void Folder::bubbleUpSyncResult() +void Folder::showSyncResultPopup() { - // count new, removed and updated items - int newItems = 0; - int removedItems = 0; - int updatedItems = 0; - int ignoredItems = 0; - int renamedItems = 0; - int conflictItems = 0; - int errorItems = 0; - - SyncFileItemPtr firstItemNew; - SyncFileItemPtr firstItemDeleted; - SyncFileItemPtr firstItemUpdated; - SyncFileItemPtr firstItemRenamed; - SyncFileItemPtr firstConflictItem; - SyncFileItemPtr firstItemError; - - QElapsedTimer timer; - timer.start(); - - foreach (const SyncFileItemPtr &item, _syncResult.syncFileItemVector() ) { - // Process the item to the gui - if( item->_status == SyncFileItem::FatalError || item->_status == SyncFileItem::NormalError ) { - //: this displays an error string (%2) for a file %1 - slotSyncError( tr("%1: %2").arg(item->_file, item->_errorString) ); - errorItems++; - if (!firstItemError) { - firstItemError = item; - } - } else if( item->_status == SyncFileItem::FileIgnored ) { - // ignored files don't show up in notifications - continue; - } else if( item->_status == SyncFileItem::Conflict ) { - conflictItems++; - if (!firstConflictItem) { - firstConflictItem = item; - } - } else { - // add new directories or remove gone away dirs to the watcher - if (item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NEW ) { - FolderMan::instance()->addMonitorPath( alias(), path()+item->_file ); - } - if (item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_REMOVE ) { - FolderMan::instance()->removeMonitorPath( alias(), path()+item->_file ); - } - - if (!item->hasErrorStatus() && item->_direction == SyncFileItem::Down) { - switch (item->_instruction) { - case CSYNC_INSTRUCTION_NEW: - case CSYNC_INSTRUCTION_TYPE_CHANGE: - newItems++; - if (!firstItemNew) - firstItemNew = item; - break; - case CSYNC_INSTRUCTION_REMOVE: - removedItems++; - if (!firstItemDeleted) - firstItemDeleted = item; - break; - case CSYNC_INSTRUCTION_SYNC: - updatedItems++; - if (!firstItemUpdated) - firstItemUpdated = item; - break; - case CSYNC_INSTRUCTION_ERROR: - qDebug() << "Got Instruction ERROR. " << _syncResult.errorString(); - break; - case CSYNC_INSTRUCTION_RENAME: - if (!firstItemRenamed) { - firstItemRenamed = item; - } - renamedItems++; - break; - default: - // nothing. - break; - } - } else if( item->_direction == SyncFileItem::None ) { // ignored files counting. - if( item->_instruction == CSYNC_INSTRUCTION_IGNORE ) { - ignoredItems++; - } - } - } + if( _syncResult.firstItemNew() ) { + createGuiLog( _syncResult.firstItemNew()->_file, LogStatusNew, _syncResult.numNewItems() ); + } + if( _syncResult.firstItemDeleted() ) { + createGuiLog( _syncResult.firstItemDeleted()->_file, LogStatusRemove, _syncResult.numRemovedItems() ); + } + if( _syncResult.firstItemUpdated() ) { + createGuiLog( _syncResult.firstItemUpdated()->_file, LogStatusUpdated, _syncResult.numUpdatedItems() ); } - qDebug() << "Processing result list and logging took " << timer.elapsed() << " Milliseconds."; - _syncResult.setWarnCount(ignoredItems); - - if( firstItemNew ) { - createGuiLog( firstItemNew->_file, LogStatusNew, newItems ); - } - if( firstItemDeleted ) { - createGuiLog( firstItemDeleted->_file, LogStatusRemove, removedItems ); - } - if( firstItemUpdated ) { - createGuiLog( firstItemUpdated->_file, LogStatusUpdated, updatedItems ); - } - - if( firstItemRenamed ) { + if( _syncResult.firstItemRenamed() ) { LogStatus status(LogStatusRename); // if the path changes it's rather a move - QDir renTarget = QFileInfo(firstItemRenamed->_renameTarget).dir(); - QDir renSource = QFileInfo(firstItemRenamed->_file).dir(); + QDir renTarget = QFileInfo(_syncResult.firstItemRenamed()->_renameTarget).dir(); + QDir renSource = QFileInfo(_syncResult.firstItemRenamed()->_file).dir(); if(renTarget != renSource) { status = LogStatusMove; } - createGuiLog( firstItemRenamed->_originalFile, status, renamedItems, firstItemRenamed->_renameTarget ); + createGuiLog( _syncResult.firstItemRenamed()->_originalFile, status, _syncResult.numRenamedItems(), _syncResult.firstItemRenamed()->_renameTarget ); } - if( firstConflictItem ) { - createGuiLog( firstConflictItem->_file, LogStatusConflict, conflictItems ); + if( _syncResult.firstConflictItem() ) { + createGuiLog( _syncResult.firstConflictItem()->_file, LogStatusConflict, _syncResult.numConflictItems() ); } - createGuiLog( firstItemError->_file, LogStatusError, errorItems ); + createGuiLog( _syncResult.firstItemError()->_file, LogStatusError, _syncResult.numErrorItems() ); qDebug() << "OO folder slotSyncFinished: result: " << int(_syncResult.status()); } @@ -574,12 +484,6 @@ void Folder::slotWatchedPathChanged(const QString& path) scheduleThisFolderSoon(); } -void Folder::slotThreadTreeWalkResult(const SyncFileItemVector& items) -{ - _syncResult.setSyncFileItemVector(items); -} - - void Folder::saveToSettings() const { // Remove first to make sure we don't get duplicates @@ -642,9 +546,6 @@ void Folder::slotTerminateSync() if( _engine->isSyncRunning() ) { _engine->abort(); - // Do not display an error message, user knows his own actions. - // _errors.append( tr("The CSync thread terminated.") ); - // _csyncError = true; setSyncState(SyncResult::SyncAbortRequested); } } @@ -725,14 +626,10 @@ void Folder::startSync(const QStringList &pathList) qCritical() << "* ERROR csync is still running and new sync requested."; return; } - _errors.clear(); - _csyncError = false; _csyncUnavail = false; _timeSinceLastSyncStart.restart(); - _syncResult.clearErrors(); _syncResult.setStatus( SyncResult::SyncPrepare ); - _syncResult.setSyncFileItemVector(SyncFileItemVector()); emit syncStateChange(); qDebug() << "*** Start syncing " << remoteUrl().toString() << " - client version" @@ -786,8 +683,7 @@ void Folder::setDirtyNetworkLimits() void Folder::slotSyncError(const QString& err) { - _errors.append( err ); - _csyncError = true; + _syncResult.appendErrorString(err); } void Folder::slotSyncStarted() @@ -809,29 +705,26 @@ void Folder::slotSyncFinished(bool success) #if QT_VERSION >= QT_VERSION_CHECK(5,0,0) << " SSL " << QSslSocket::sslLibraryVersionString().toUtf8().data() #endif - ; + ; - - if( _csyncError ) { - qDebug() << "-> SyncEngine finished with ERROR, warn count is" << _syncResult.warnCount(); + bool syncError = !_syncResult.errorStrings().isEmpty(); + if( syncError ) { + qDebug() << "-> SyncEngine finished with ERROR"; } else { qDebug() << "-> SyncEngine finished without problem."; } _fileLog->finish(); - bubbleUpSyncResult(); + showSyncResultPopup(); auto anotherSyncNeeded = _engine->isAnotherSyncNeeded(); - if (_csyncError) { + if (syncError) { _syncResult.setStatus(SyncResult::Error); - qDebug() << " ** error Strings: " << _errors; - _syncResult.setErrorStrings( _errors ); qDebug() << " * owncloud csync thread finished with error"; } else if (_csyncUnavail) { _syncResult.setStatus(SyncResult::Error); qDebug() << " ** csync not available."; - } else if( _syncResult.warnCount() > 0 ) { - // there have been warnings on the way. + } else if( _syncResult.foundFilesNotSynced() ) { _syncResult.setStatus(SyncResult::Problem); } else if( _definition.paused ) { // Maybe the sync was terminated because the user paused the folder @@ -908,10 +801,6 @@ void Folder::slotFolderDiscovered(bool, QString folderName) // and hand the result over to the progress dispatcher. void Folder::slotTransmissionProgress(const ProgressInfo &pi) { - if( !pi.isUpdatingEstimates() ) { - // this is the beginning of a sync, set the warning level to 0 - _syncResult.setWarnCount(0); - } emit progressInfo(pi); ProgressDispatcher::instance()->setProgressInfo(alias(), pi); } @@ -919,10 +808,16 @@ void Folder::slotTransmissionProgress(const ProgressInfo &pi) // a item is completed: count the errors and forward to the ProgressDispatcher void Folder::slotItemCompleted(const SyncFileItemPtr &item) { - if (Progress::isWarningKind(item->_status)) { - // Count all error conditions. - _syncResult.setWarnCount(_syncResult.warnCount()+1); + // add new directories or remove gone away dirs to the watcher + if (item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NEW ) { + FolderMan::instance()->addMonitorPath( alias(), path()+item->_file ); } + if (item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_REMOVE ) { + FolderMan::instance()->removeMonitorPath( alias(), path()+item->_file ); + } + + _syncResult.processCompletedItem(item); + _fileLog->logItem(*item); emit ProgressDispatcher::instance()->itemCompleted(alias(), item); } diff --git a/src/gui/folder.h b/src/gui/folder.h index eb9e3ca75..6c4b73c2d 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -286,8 +286,6 @@ private slots: void etagRetreived(const QString &); void etagRetreivedFromSyncEngine(const QString &); - void slotThreadTreeWalkResult(const SyncFileItemVector& ); // after sync is done - void slotEmitFinishedDelayed(); void slotNewBigFolderDiscovered(const QString &); @@ -302,7 +300,7 @@ private slots: private: bool setIgnoredFiles(); - void bubbleUpSyncResult(); + void showSyncResultPopup(); void checkLocalPath(); @@ -325,8 +323,6 @@ private: SyncResult _syncResult; QScopedPointer _engine; - QStringList _errors; - bool _csyncError; bool _csyncUnavail; bool _wipeDb; bool _proxyDirty; diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index 067f7fda9..dbe996792 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -1121,7 +1121,7 @@ void FolderMan::setDirtyNetworkLimits() SyncResult FolderMan::accountStatus(const QList &folders) { - SyncResult overallResult(SyncResult::Undefined); + SyncResult overallResult; int cnt = folders.count(); @@ -1235,10 +1235,10 @@ SyncResult FolderMan::accountStatus(const QList &folders) return overallResult; } -QString FolderMan::statusToString( SyncResult syncStatus, bool paused ) const +QString FolderMan::statusToString( SyncResult::Status syncStatus, bool paused ) const { QString folderMessage; - switch( syncStatus.status() ) { + switch( syncStatus ) { case SyncResult::Undefined: folderMessage = tr( "Undefined State." ); break; diff --git a/src/gui/folderman.h b/src/gui/folderman.h index 911db7089..7386335b4 100644 --- a/src/gui/folderman.h +++ b/src/gui/folderman.h @@ -106,7 +106,7 @@ public: /** Creates a new and empty local directory. */ bool startFromScratch( const QString& ); - QString statusToString(SyncResult, bool paused ) const; + QString statusToString(SyncResult::Status, bool paused ) const; static SyncResult accountStatus( const QList &folders ); diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index 2aaa4658f..c8b5c4925 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -1012,18 +1012,10 @@ void FolderStatusModel::slotFolderSyncStateChange(Folder *f) // update the icon etc. now slotUpdateFolderState(f); - if (state == SyncResult::Success) { - foreach (const SyncFileItemPtr &i, f->syncResult().syncFileItemVector()) { - if (i->_isDirectory && (i->_instruction == CSYNC_INSTRUCTION_NEW - || i->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE - || i->_instruction == CSYNC_INSTRUCTION_REMOVE - || i->_instruction == CSYNC_INSTRUCTION_RENAME)) { - // There is a new or a removed folder. reset all data - auto & info = _folders[folderIndex]; - info.resetSubs(this, index(folderIndex)); - return; - } - } + if (state == SyncResult::Success && f->syncResult().folderStructureWasChanged()) { + // There is a new or a removed folder. reset all data + auto & info = _folders[folderIndex]; + info.resetSubs(this, index(folderIndex)); } } diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 20535345e..ce44878cc 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -268,11 +268,11 @@ bool SyncEngine::checkErrorBlacklisting( SyncFileItem &item ) return true; } -void SyncEngine::deleteStaleDownloadInfos() +void SyncEngine::deleteStaleDownloadInfos(const SyncFileItemVector &syncItems) { // Find all downloadinfo paths that we want to preserve. QSet download_file_paths; - foreach(const SyncFileItemPtr &it, _syncedItems) { + foreach(const SyncFileItemPtr &it, syncItems) { if (it->_direction == SyncFileItem::Down && it->_type == SyncFileItem::File) { @@ -290,11 +290,11 @@ void SyncEngine::deleteStaleDownloadInfos() } } -void SyncEngine::deleteStaleUploadInfos() +void SyncEngine::deleteStaleUploadInfos(const SyncFileItemVector &syncItems) { // Find all blacklisted paths that we want to preserve. QSet upload_file_paths; - foreach(const SyncFileItemPtr &it, _syncedItems) { + foreach(const SyncFileItemPtr &it, syncItems) { if (it->_direction == SyncFileItem::Up && it->_type == SyncFileItem::File) { @@ -315,11 +315,11 @@ void SyncEngine::deleteStaleUploadInfos() } } -void SyncEngine::deleteStaleErrorBlacklistEntries() +void SyncEngine::deleteStaleErrorBlacklistEntries(const SyncFileItemVector &syncItems) { // Find all blacklisted paths that we want to preserve. QSet blacklist_file_paths; - foreach(const SyncFileItemPtr &it, _syncedItems) { + foreach(const SyncFileItemPtr &it, syncItems) { if (it->_hasBlacklistEntry) blacklist_file_paths.insert(it->_file); } @@ -754,7 +754,6 @@ void SyncEngine::startSync() qDebug() << "Could not determine free space available at" << _localPath; } - _syncedItems.clear(); _syncItemMap.clear(); _needsUpdate = false; @@ -922,12 +921,12 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) csync_commit(_csync_ctx); // The map was used for merging trees, convert it to a list: - _syncedItems = _syncItemMap.values().toVector(); + SyncFileItemVector syncItems = _syncItemMap.values().toVector(); _syncItemMap.clear(); // free memory // Adjust the paths for the renames. - for (SyncFileItemVector::iterator it = _syncedItems.begin(); - it != _syncedItems.end(); ++it) { + for (SyncFileItemVector::iterator it = syncItems.begin(); + it != syncItems.end(); ++it) { (*it)->_file = adjustRenamedPath((*it)->_file); } @@ -935,7 +934,7 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) if (_account->serverVersionInt() < 0x080100) { // Server version older than 8.1 don't support these character in filename. static const QRegExp invalidCharRx("[\\\\:?*\"<>|]"); - for (auto it = _syncedItems.begin(); it != _syncedItems.end(); ++it) { + for (auto it = syncItems.begin(); it != syncItems.end(); ++it) { if ((*it)->_direction == SyncFileItem::Up && (*it)->destination().contains(invalidCharRx)) { (*it)->_errorString = tr("File name contains at least one invalid character"); @@ -947,7 +946,7 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) if (!_hasNoneFiles && _hasRemoveFile) { qDebug() << Q_FUNC_INFO << "All the files are going to be changed, asking the user"; bool cancel = false; - emit aboutToRemoveAllFiles(_syncedItems.first()->_direction, &cancel); + emit aboutToRemoveAllFiles(syncItems.first()->_direction, &cancel); if (cancel) { qDebug() << Q_FUNC_INFO << "Abort sync"; finalize(false); @@ -962,7 +961,7 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) if (!databaseFingerprint.isNull() && _discoveryMainThread->_dataFingerprint != databaseFingerprint) { qDebug() << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryMainThread->_dataFingerprint; - restoreOldFiles(); + restoreOldFiles(syncItems); } else if (!_hasForwardInTimeFiles && _backInTimeFiles >= 2 && _account->serverVersionInt() < 0x090100) { // The server before ownCloud 9.1 did not have the data-fingerprint property. So in that // case we use heuristics to detect restored backup. This is disabled with newer version @@ -972,18 +971,18 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) bool restore = false; emit aboutToRestoreBackup(&restore); if (restore) { - restoreOldFiles(); + restoreOldFiles(syncItems); } } // Sort items per destination - std::sort(_syncedItems.begin(), _syncedItems.end()); + std::sort(syncItems.begin(), syncItems.end()); // make sure everything is allowed - checkForPermission(); + checkForPermission(syncItems); // To announce the beginning of the sync - emit aboutToPropagate(_syncedItems); + emit aboutToPropagate(syncItems); // it's important to do this before ProgressInfo::start(), to announce start of new sync emit transmissionProgress(*_progressInfo); _progressInfo->startEstimateUpdates(); @@ -1016,16 +1015,16 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) // apply the network limits to the propagator setNetworkLimits(_uploadLimit, _downloadLimit); - deleteStaleDownloadInfos(); - deleteStaleUploadInfos(); - deleteStaleErrorBlacklistEntries(); + deleteStaleDownloadInfos(syncItems); + deleteStaleUploadInfos(syncItems); + deleteStaleErrorBlacklistEntries(syncItems); _journal->commit("post stale entry removal"); // Emit the started signal only after the propagator has been set up. if (_needsUpdate) emit(started()); - _propagator->start(_syncedItems); + _propagator->start(syncItems); qDebug() << "<<#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Post-Reconcile Finished")); } @@ -1098,7 +1097,6 @@ void SyncEngine::slotFinished(bool success) // files needed propagation emit transmissionProgress(*_progressInfo); - emit treeWalkResult(_syncedItems); finalize(success); } @@ -1148,13 +1146,13 @@ QString SyncEngine::adjustRenamedPath(const QString& original) * Make sure that we are allowed to do what we do by checking the permissions and the selective sync list * */ -void SyncEngine::checkForPermission() +void SyncEngine::checkForPermission(SyncFileItemVector &syncItems) { bool selectiveListOk; auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &selectiveListOk); std::sort(selectiveSyncBlackList.begin(), selectiveSyncBlackList.end()); - for (SyncFileItemVector::iterator it = _syncedItems.begin(); it != _syncedItems.end(); ++it) { + for (SyncFileItemVector::iterator it = syncItems.begin(); it != syncItems.end(); ++it) { if ((*it)->_direction != SyncFileItem::Up) { // Currently we only check server-side permissions continue; @@ -1171,7 +1169,7 @@ void SyncEngine::checkForPermission() (*it)->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist"); if ((*it)->_isDirectory) { - for (SyncFileItemVector::iterator it_next = it + 1; it_next != _syncedItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { + for (SyncFileItemVector::iterator it_next = it + 1; it_next != syncItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { it = it_next; (*it)->_instruction = CSYNC_INSTRUCTION_IGNORE; (*it)->_status = SyncFileItem::FileIgnored; @@ -1196,7 +1194,7 @@ void SyncEngine::checkForPermission() (*it)->_status = SyncFileItem::NormalError; (*it)->_errorString = tr("Not allowed because you don't have permission to add subfolders to that folder"); - for (SyncFileItemVector::iterator it_next = it + 1; it_next != _syncedItems.end() && (*it_next)->destination().startsWith(path); ++it_next) { + for (SyncFileItemVector::iterator it_next = it + 1; it_next != syncItems.end() && (*it_next)->destination().startsWith(path); ++it_next) { it = it_next; if ((*it)->_instruction == CSYNC_INSTRUCTION_RENAME) { // The file was most likely moved in this directory. @@ -1256,7 +1254,7 @@ void SyncEngine::checkForPermission() if ((*it)->_isDirectory) { // restore all sub items for (SyncFileItemVector::iterator it_next = it + 1; - it_next != _syncedItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { + it_next != syncItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { it = it_next; if ((*it)->_instruction != CSYNC_INSTRUCTION_REMOVE) { @@ -1282,12 +1280,12 @@ void SyncEngine::checkForPermission() // underneath, propagator sees that. if( (*it)->_isDirectory ) { // put a more descriptive message if a top level share dir really is removed. - if( it == _syncedItems.begin() || !(path.startsWith((*(it-1))->_file)) ) { + if( it == syncItems.begin() || !(path.startsWith((*(it-1))->_file)) ) { (*it)->_errorString = tr("Local files and share folder removed."); } for (SyncFileItemVector::iterator it_next = it + 1; - it_next != _syncedItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { + it_next != syncItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { it = it_next; } } @@ -1366,7 +1364,7 @@ void SyncEngine::checkForPermission() if ((*it)->_isDirectory) { for (SyncFileItemVector::iterator it_next = it + 1; - it_next != _syncedItems.end() && (*it_next)->destination().startsWith(path); ++it_next) { + it_next != syncItems.end() && (*it_next)->destination().startsWith(path); ++it_next) { it = it_next; (*it)->_instruction = CSYNC_INSTRUCTION_ERROR; (*it)->_status = SyncFileItem::NormalError; @@ -1395,7 +1393,7 @@ QByteArray SyncEngine::getPermissions(const QString& file) const return _remotePerms.value(file); } -void SyncEngine::restoreOldFiles() +void SyncEngine::restoreOldFiles(SyncFileItemVector &syncItems) { /* When the server is trying to send us lots of file in the past, this means that a backup was restored in the server. In that case, we should not simply overwrite the newer file @@ -1403,7 +1401,7 @@ void SyncEngine::restoreOldFiles() upload the client file. But we still downloaded the old file in a conflict file just in case */ - for (auto it = _syncedItems.begin(); it != _syncedItems.end(); ++it) { + for (auto it = syncItems.begin(); it != syncItems.end(); ++it) { if ((*it)->_direction != SyncFileItem::Down) continue; diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 52f695a1a..b203a9e80 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -123,9 +123,6 @@ signals: // after each item completed by a job (successful or not) void itemCompleted(const SyncFileItemPtr&); - // after sync is done - void treeWalkResult(const SyncFileItemVector&); - void transmissionProgress( const ProgressInfo& progress ); void finished(bool success); @@ -179,13 +176,13 @@ private: // Cleans up unnecessary downloadinfo entries in the journal as well // as their temporary files. - void deleteStaleDownloadInfos(); + void deleteStaleDownloadInfos(const SyncFileItemVector &syncItems); // Removes stale uploadinfos from the journal. - void deleteStaleUploadInfos(); + void deleteStaleUploadInfos(const SyncFileItemVector &syncItems); // Removes stale error blacklist entries from the journal. - void deleteStaleErrorBlacklistEntries(); + void deleteStaleErrorBlacklistEntries(const SyncFileItemVector &syncItems); // cleanup and emit the finished signal void finalize(bool success); @@ -195,10 +192,6 @@ private: // Must only be acessed during update and reconcile QMap _syncItemMap; - // should be called _syncItems (present tense). It's the items from the _syncItemMap but - // sorted and re-adjusted based on permissions. - SyncFileItemVector _syncedItems; - AccountPtr _account; CSYNC *_csync_ctx; bool _needsUpdate; @@ -240,13 +233,13 @@ private: * check if we are allowed to propagate everything, and if we are not, adjust the instructions * to recover */ - void checkForPermission(); + void checkForPermission(SyncFileItemVector &syncItems); QByteArray getPermissions(const QString& file) const; /** * Instead of downloading files from the server, upload the files to the server */ - void restoreOldFiles(); + void restoreOldFiles(SyncFileItemVector &syncItems); bool _hasNoneFiles; // true if there is at least one file which was not changed on the server bool _hasRemoveFile; // true if there is at leasr one file with instruction REMOVE diff --git a/src/libsync/syncresult.cpp b/src/libsync/syncresult.cpp index acefc9735..625ebae2c 100644 --- a/src/libsync/syncresult.cpp +++ b/src/libsync/syncresult.cpp @@ -13,19 +13,22 @@ */ #include "syncresult.h" +#include "progressdispatcher.h" namespace OCC { SyncResult::SyncResult() - : _status( Undefined ), - _warnCount(0) -{ -} + : _status( Undefined ) + , _foundFilesNotSynced(false) + , _folderStructureWasChanged(false) + , _numNewItems(0) + , _numRemovedItems(0) + , _numUpdatedItems(0) + , _numRenamedItems(0) + , _numConflictItems(0) + , _numErrorItems(0) -SyncResult::SyncResult(SyncResult::Status status ) - : _status(status), - _warnCount(0) { } @@ -34,6 +37,11 @@ SyncResult::Status SyncResult::status() const return _status; } +void SyncResult::reset() +{ + *this = SyncResult(); +} + QString SyncResult::statusString() const { QString re; @@ -80,42 +88,17 @@ void SyncResult::setStatus( Status stat ) _syncTime = QDateTime::currentDateTime(); } -void SyncResult::setSyncFileItemVector( const SyncFileItemVector& items ) -{ - _syncItems = items; -} - -SyncFileItemVector SyncResult::syncFileItemVector() const -{ - return _syncItems; -} - QDateTime SyncResult::syncTime() const { return _syncTime; } -void SyncResult::setWarnCount(int wc) -{ - _warnCount = wc; -} - -int SyncResult::warnCount() const -{ - return _warnCount; -} - -void SyncResult::setErrorStrings( const QStringList& list ) -{ - _errors = list; -} - QStringList SyncResult::errorStrings() const { return _errors; } -void SyncResult::setErrorString( const QString& err ) +void SyncResult::appendErrorString( const QString& err ) { _errors.append( err ); } @@ -141,8 +124,68 @@ QString SyncResult::folder() const return _folder; } -SyncResult::~SyncResult() +void SyncResult::processCompletedItem(const SyncFileItemPtr &item) { + if (Progress::isWarningKind(item->_status)) { + // Count any error conditions, error strings will have priority anyway. + _foundFilesNotSynced = true; + } + + if (item->_isDirectory && (item->_instruction == CSYNC_INSTRUCTION_NEW + || item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE + || item->_instruction == CSYNC_INSTRUCTION_REMOVE + || item->_instruction == CSYNC_INSTRUCTION_RENAME)) { + _folderStructureWasChanged = true; + } + + // Process the item to the gui + if( item->_status == SyncFileItem::FatalError || item->_status == SyncFileItem::NormalError ) { + //: this displays an error string (%2) for a file %1 + appendErrorString( QObject::tr("%1: %2").arg(item->_file, item->_errorString) ); + _numErrorItems++; + if (!_firstItemError) { + _firstItemError = item; + } + } else if( item->_status == SyncFileItem::Conflict ) { + _numConflictItems++; + if (!_firstConflictItem) { + _firstConflictItem = item; + } + } else { + if (!item->hasErrorStatus() && item->_status != SyncFileItem::FileIgnored && item->_direction == SyncFileItem::Down) { + switch (item->_instruction) { + case CSYNC_INSTRUCTION_NEW: + case CSYNC_INSTRUCTION_TYPE_CHANGE: + _numNewItems++; + if (!_firstItemNew) + _firstItemNew = item; + break; + case CSYNC_INSTRUCTION_REMOVE: + _numRemovedItems++; + if (!_firstItemDeleted) + _firstItemDeleted = item; + break; + case CSYNC_INSTRUCTION_SYNC: + _numUpdatedItems++; + if (!_firstItemUpdated) + _firstItemUpdated = item; + break; + case CSYNC_INSTRUCTION_RENAME: + if (!_firstItemRenamed) { + _firstItemRenamed = item; + } + _numRenamedItems++; + break; + default: + // nothing. + break; + } + } else if( item->_direction == SyncFileItem::None ) { + if( item->_instruction == CSYNC_INSTRUCTION_IGNORE ) { + _foundFilesNotSynced = true; + } + } + } } diff --git a/src/libsync/syncresult.h b/src/libsync/syncresult.h index 14e062ca3..0c630f737 100644 --- a/src/libsync/syncresult.h +++ b/src/libsync/syncresult.h @@ -47,20 +47,13 @@ public: }; SyncResult(); - SyncResult( Status status ); - ~SyncResult(); - void setErrorString( const QString& ); - void setErrorStrings( const QStringList& ); + void reset(); + + void appendErrorString( const QString& ); QString errorString() const; QStringList errorStrings() const; - int warnCount() const; - void setWarnCount(int wc); void clearErrors(); - // handle a list of changed items. - void setSyncFileItemVector( const SyncFileItemVector& ); - SyncFileItemVector syncFileItemVector() const; - void setStatus( Status ); Status status() const; QString statusString() const; @@ -68,6 +61,25 @@ public: void setFolder(const QString& folder); QString folder() const; + bool foundFilesNotSynced() const { return _foundFilesNotSynced; } + bool folderStructureWasChanged() const { return _folderStructureWasChanged; } + + int numNewItems() const { return _numNewItems; } + int numRemovedItems() const { return _numRemovedItems; } + int numUpdatedItems() const { return _numUpdatedItems; } + int numRenamedItems() const { return _numRenamedItems; } + int numConflictItems() const { return _numConflictItems; } + int numErrorItems() const { return _numErrorItems; } + + const SyncFileItemPtr& firstItemNew() const { return _firstItemNew; } + const SyncFileItemPtr& firstItemDeleted() const { return _firstItemDeleted; } + const SyncFileItemPtr& firstItemUpdated() const { return _firstItemUpdated; } + const SyncFileItemPtr& firstItemRenamed() const { return _firstItemRenamed; } + const SyncFileItemPtr& firstConflictItem() const { return _firstConflictItem; } + const SyncFileItemPtr& firstItemError() const { return _firstItemError; } + + void processCompletedItem(const SyncFileItemPtr &item); + private: Status _status; SyncFileItemVector _syncItems; @@ -77,7 +89,23 @@ private: * when the sync tool support this... */ QStringList _errors; - int _warnCount; + bool _foundFilesNotSynced; + bool _folderStructureWasChanged; + + // count new, removed and updated items + int _numNewItems; + int _numRemovedItems; + int _numUpdatedItems; + int _numRenamedItems; + int _numConflictItems; + int _numErrorItems; + + SyncFileItemPtr _firstItemNew; + SyncFileItemPtr _firstItemDeleted; + SyncFileItemPtr _firstItemUpdated; + SyncFileItemPtr _firstItemRenamed; + SyncFileItemPtr _firstConflictItem; + SyncFileItemPtr _firstItemError; }; }