From 96ecdb866d0142619fbf17274d6b71cc2e91076e Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Fri, 30 Jan 2015 13:36:20 +0100 Subject: [PATCH] Time estimate: Refactor remaining time guess. #2328 --- src/cmd/cmd.cpp | 2 +- src/gui/accountsettings.cpp | 25 +-- src/gui/accountsettings.h | 2 +- src/gui/application.cpp | 2 - src/gui/folder.cpp | 10 +- src/gui/folder.h | 2 +- src/gui/owncloudgui.cpp | 22 +-- src/gui/owncloudgui.h | 2 +- src/gui/protocolwidget.cpp | 12 +- src/gui/protocolwidget.h | 2 +- src/gui/settingsdialog.cpp | 4 +- src/libsync/progressdispatcher.cpp | 191 ++++++++++++++++++- src/libsync/progressdispatcher.h | 285 +++++++++++++++-------------- src/libsync/syncengine.cpp | 31 ++-- src/libsync/syncengine.h | 4 +- 15 files changed, 396 insertions(+), 200 deletions(-) diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp index 55d924b6e..486cef3b2 100644 --- a/src/cmd/cmd.cpp +++ b/src/cmd/cmd.cpp @@ -471,7 +471,7 @@ restart_sync: SyncEngine engine(account, _csync_ctx, options.source_dir, QUrl(options.target_url).path(), folder, &db); QObject::connect(&engine, SIGNAL(finished()), &app, SLOT(quit())); - QObject::connect(&engine, SIGNAL(transmissionProgress(Progress::Info)), &cmd, SLOT(transmissionProgressSlot())); + QObject::connect(&engine, SIGNAL(transmissionProgress(ProgressInfo)), &cmd, SLOT(transmissionProgressSlot())); engine.setSelectiveSyncBlackList(selectiveSyncList); diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index ac195cc88..4a7baf97c 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -593,7 +593,7 @@ QString AccountSettings::shortenFilename( const QString& folder, const QString& return shortFile; } -void AccountSettings::slotSetProgress(const QString& folder, const Progress::Info &progress ) +void AccountSettings::slotSetProgress(const QString& folder, const ProgressInfo &progress ) { if (!isVisible()) { return; // for https://github.com/owncloud/client/issues/2648#issuecomment-71377909 @@ -630,10 +630,10 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf SyncFileItem curItem = progress._lastCompletedItem; qint64 curItemProgress = -1; // -1 means finished quint64 biggerItemSize = -1; - foreach(const Progress::Info::ProgressItem &citm, progress._currentItems) { - if (curItemProgress == -1 || (Progress::isSizeDependent(citm._item) + foreach(const ProgressInfo::ProgressItem &citm, progress._currentItems) { + if (curItemProgress == -1 || (ProgressInfo::isSizeDependent(citm._item) && biggerItemSize < citm._item._size)) { - curItemProgress = citm._completedSize; + curItemProgress = citm._progress.completed(); curItem = citm._item; biggerItemSize = citm._item._size; } @@ -648,15 +648,15 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf QString fileProgressString; - if (Progress::isSizeDependent(curItem)) { + if (ProgressInfo::isSizeDependent(curItem)) { QString s1 = Utility::octetsToString( curItemProgress ); QString s2 = Utility::octetsToString( curItem._size ); - quint64 estimatedBw = progress.getFileEstimate(curItem).getEstimatedBandwidth(); + quint64 estimatedBw = progress.fileProgress(curItem).estimatedBandwidth; if (estimatedBw) { //: Example text: "uploading foobar.png (1MB of 2MB) time left 2 minutes at a rate of 24Kb/s" fileProgressString = tr("%1 %2 (%3 of %4) %5 left at a rate of %6/s") .arg(kindString, itemFileName, s1, s2, - Utility::timeToDescriptiveString(progress.getFileEstimate(curItem).getEtaEstimate(), 3, " ", true), + Utility::timeToDescriptiveString(progress.fileProgress(curItem).estimatedEta, 3, " ", true), Utility::octetsToString(estimatedBw) ); } else { //: Example text: "uploading foobar.png (2MB of 2MB)" @@ -670,11 +670,12 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf // overall progress quint64 completedSize = progress.completedSize(); - quint64 currentFile = progress._completedFileCount + progress._currentItems.count(); + quint64 completedFile = progress.completedFiles(); + quint64 currentFile = progress.currentFile(); if (currentFile == ULLONG_MAX) currentFile = 0; - quint64 totalSize = qMax(completedSize, progress._totalSize); - quint64 totalFileCount = qMax(currentFile, progress._totalFileCount); + quint64 totalSize = qMax(completedSize, progress.totalSize()); + quint64 totalFileCount = qMax(currentFile, progress.totalFiles()); QString overallSyncString; if (totalSize > 0) { QString s1 = Utility::octetsToString( completedSize ); @@ -682,7 +683,7 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf overallSyncString = tr("%1 of %2, file %3 of %4\nTotal time left %5") .arg(s1, s2) .arg(currentFile).arg(totalFileCount) - .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 3, " ", true) ); + .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 3, " ", true) ); } else if (totalFileCount > 0) { // Don't attemt to estimate the time left if there is no kb to transfer. overallSyncString = tr("file %1 of %2") .arg(currentFile).arg(totalFileCount); @@ -693,7 +694,7 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf int overallPercent = 0; if( totalFileCount > 0 ) { // Add one 'byte' for each files so the percentage is moving when deleting or renaming files - overallPercent = qRound(double(completedSize + progress._completedFileCount)/double(totalSize + totalFileCount) * 100.0); + overallPercent = qRound(double(completedSize + completedFile)/double(totalSize + totalFileCount) * 100.0); } overallPercent = qBound(0, overallPercent, 100); item->setData( overallPercent, FolderStatusDelegate::SyncProgressOverallPercent); diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index 46b6fd9ba..a6811eb1f 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -61,7 +61,7 @@ public slots: void slotOpenOC(); void slotUpdateFolderState( Folder* ); void slotDoubleClicked( const QModelIndex& ); - void slotSetProgress(const QString& folder, const Progress::Info& progress); + void slotSetProgress(const QString& folder, const ProgressInfo& progress); void slotButtonsSetEnabled(); void slotUpdateQuota( qint64,qint64 ); diff --git a/src/gui/application.cpp b/src/gui/application.cpp index 946384ea1..5898d76e0 100644 --- a/src/gui/application.cpp +++ b/src/gui/application.cpp @@ -136,8 +136,6 @@ Application::Application(int &argc, char **argv) : setQuitOnLastWindowClosed(false); - qRegisterMetaType("Progress::Info"); - ConfigFile cfg; _theme->setSystrayUseMonoIcons(cfg.monoIcons()); connect (_theme, SIGNAL(systrayUseMonoIconsChanged(bool)), SLOT(slotUseMonoIconsChanged(bool))); diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 96713d0cd..0e55b8cd9 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -804,7 +804,7 @@ void Folder::startSync(const QStringList &pathList) connect(_engine.data(), SIGNAL(aboutToRemoveAllFiles(SyncFileItem::Direction,bool*)), SLOT(slotAboutToRemoveAllFiles(SyncFileItem::Direction,bool*))); connect(_engine.data(), SIGNAL(folderDiscovered(bool,QString)), this, SLOT(slotFolderDiscovered(bool,QString))); - connect(_engine.data(), SIGNAL(transmissionProgress(Progress::Info)), this, SLOT(slotTransmissionProgress(Progress::Info))); + connect(_engine.data(), SIGNAL(transmissionProgress(ProgressInfo)), this, SLOT(slotTransmissionProgress(ProgressInfo))); connect(_engine.data(), SIGNAL(jobCompleted(const SyncFileItem &)), this, SLOT(slotJobCompleted(const SyncFileItem &))); connect(_engine.data(), SIGNAL(syncItemDiscovered(const SyncFileItem &)), this, SLOT(slotSyncItemDiscovered(const SyncFileItem &))); @@ -959,7 +959,7 @@ void Folder::slotEmitFinishedDelayed() void Folder::slotFolderDiscovered(bool, QString folderName) { - Progress::Info pi; + ProgressInfo pi; pi._currentDiscoveredFolder = folderName; ProgressDispatcher::instance()->setProgressInfo(alias(), pi); } @@ -967,10 +967,10 @@ void Folder::slotFolderDiscovered(bool, QString folderName) // the progress comes without a folder and the valid path set. Add that here // and hand the result over to the progress dispatcher. -void Folder::slotTransmissionProgress(const Progress::Info &pi) +void Folder::slotTransmissionProgress(const ProgressInfo &pi) { - if( pi._completedFileCount ) { - // No job completed yet, this is the beginning of a sync, set the warning level to 0 + if( !pi.hasStarted() ) { + // this is the beginning of a sync, set the warning level to 0 _syncResult.setWarnCount(0); } ProgressDispatcher::instance()->setProgressInfo(alias(), pi); diff --git a/src/gui/folder.h b/src/gui/folder.h index f07be9570..ae299739c 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -181,7 +181,7 @@ private slots: void slotSyncFinished(); void slotFolderDiscovered(bool local, QString folderName); - void slotTransmissionProgress(const Progress::Info& pi); + void slotTransmissionProgress(const ProgressInfo& pi); void slotJobCompleted(const SyncFileItem&); void slotSyncItemDiscovered(const SyncFileItem & item); diff --git a/src/gui/owncloudgui.cpp b/src/gui/owncloudgui.cpp index 48ec2b526..63bccad4a 100644 --- a/src/gui/owncloudgui.cpp +++ b/src/gui/owncloudgui.cpp @@ -79,8 +79,8 @@ ownCloudGui::ownCloudGui(Application *parent) : this, SLOT(slotOpenPath(QString))); ProgressDispatcher *pd = ProgressDispatcher::instance(); - connect( pd, SIGNAL(progressInfo(QString,Progress::Info)), this, - SLOT(slotUpdateProgress(QString,Progress::Info)) ); + connect( pd, SIGNAL(progressInfo(QString,ProgressInfo)), this, + SLOT(slotUpdateProgress(QString,ProgressInfo)) ); FolderMan *folderMan = FolderMan::instance(); connect( folderMan, SIGNAL(folderSyncStateChange(QString)), @@ -469,24 +469,24 @@ void ownCloudGui::slotRebuildRecentMenus() } -void ownCloudGui::slotUpdateProgress(const QString &folder, const Progress::Info& progress) +void ownCloudGui::slotUpdateProgress(const QString &folder, const ProgressInfo& progress) { Q_UNUSED(folder); if (!progress._currentDiscoveredFolder.isEmpty()) { _actionStatus->setText( tr("Discovering '%1'") .arg( progress._currentDiscoveredFolder )); - } else if (progress._totalSize == 0 ) { - quint64 currentFile = progress._completedFileCount + progress._currentItems.count(); - quint64 totalFileCount = qMax(progress._totalFileCount, currentFile); + } else if (progress.totalSize() == 0 ) { + quint64 currentFile = progress.currentFile(); + quint64 totalFileCount = qMax(progress.totalFiles(), currentFile); _actionStatus->setText( tr("Syncing %1 of %2 (%3 left)") .arg( currentFile ).arg( totalFileCount ) - .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 2, " ",true) ) ); + .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 2, " ",true) ) ); } else { - QString totalSizeStr = Utility::octetsToString( progress._totalSize ); + QString totalSizeStr = Utility::octetsToString( progress.totalSize() ); _actionStatus->setText( tr("Syncing %1 (%2 left)") .arg( totalSizeStr ) - .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 2, " ",true) ) ); + .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 2, " ",true) ) ); } @@ -524,8 +524,8 @@ void ownCloudGui::slotUpdateProgress(const QString &folder, const Progress::Info slotRebuildRecentMenus(); } - if (progress._completedFileCount != ULLONG_MAX - && progress._completedFileCount >= progress._totalFileCount + if (progress.hasStarted() + && progress.completedFiles() >= progress.totalFiles() && progress._currentDiscoveredFolder.isEmpty()) { QTimer::singleShot(2000, this, SLOT(slotDisplayIdle())); } diff --git a/src/gui/owncloudgui.h b/src/gui/owncloudgui.h index 9dedcc66f..45111165b 100644 --- a/src/gui/owncloudgui.h +++ b/src/gui/owncloudgui.h @@ -56,7 +56,7 @@ public slots: void slotFolderOpenAction( const QString& alias ); void slotRefreshQuotaDisplay( qint64 total, qint64 used ); void slotRebuildRecentMenus(); - void slotUpdateProgress(const QString &folder, const Progress::Info& progress); + void slotUpdateProgress(const QString &folder, const ProgressInfo& progress); void slotShowGuiMessage(const QString &title, const QString &message); void slotFoldersChanged(); void slotShowSettings(); diff --git a/src/gui/protocolwidget.cpp b/src/gui/protocolwidget.cpp index 0bccd81d2..dc1ab2934 100644 --- a/src/gui/protocolwidget.cpp +++ b/src/gui/protocolwidget.cpp @@ -40,8 +40,8 @@ ProtocolWidget::ProtocolWidget(QWidget *parent) : { _ui->setupUi(this); - connect(ProgressDispatcher::instance(), SIGNAL(progressInfo(QString,Progress::Info)), - this, SLOT(slotProgressInfo(QString,Progress::Info))); + connect(ProgressDispatcher::instance(), SIGNAL(progressInfo(QString,ProgressInfo)), + this, SLOT(slotProgressInfo(QString,ProgressInfo))); connect(_ui->_treeWidget, SIGNAL(itemActivated(QTreeWidgetItem*,int)), SLOT(slotOpenFile(QTreeWidgetItem*,int))); @@ -224,7 +224,7 @@ QTreeWidgetItem* ProtocolWidget::createCompletedTreewidgetItem(const QString& fo icon = Theme::instance()->syncStateIcon(SyncResult::Problem); } - if (Progress::isSizeDependent(item)) { + if (ProgressInfo::isSizeDependent(item)) { columns << Utility::octetsToString( item._size ); } @@ -266,13 +266,13 @@ void ProtocolWidget::computeResyncButtonEnabled() } -void ProtocolWidget::slotProgressInfo( const QString& folder, const Progress::Info& progress ) +void ProtocolWidget::slotProgressInfo( const QString& folder, const ProgressInfo& progress ) { - if( progress._completedFileCount == ULLONG_MAX ) { + if( !progress.hasStarted() ) { // The sync is restarting, clean the old items cleanIgnoreItems(folder); computeResyncButtonEnabled(); - } else if (progress._completedFileCount >= progress._totalFileCount) { + } else if (progress.completedFiles() >= progress.totalFiles()) { //Sync completed computeResyncButtonEnabled(); } diff --git a/src/gui/protocolwidget.h b/src/gui/protocolwidget.h index 0227db373..91ae119ca 100644 --- a/src/gui/protocolwidget.h +++ b/src/gui/protocolwidget.h @@ -40,7 +40,7 @@ public: ~ProtocolWidget(); public slots: - void slotProgressInfo( const QString& folder, const Progress::Info& progress ); + void slotProgressInfo( const QString& folder, const ProgressInfo& progress ); void slotOpenFile( QTreeWidgetItem* item, int ); protected slots: diff --git a/src/gui/settingsdialog.cpp b/src/gui/settingsdialog.cpp index acd80d958..2aea2b875 100644 --- a/src/gui/settingsdialog.cpp +++ b/src/gui/settingsdialog.cpp @@ -112,8 +112,8 @@ SettingsDialog::SettingsDialog(ownCloudGui *gui, QWidget *parent) : connect( _accountSettings, SIGNAL(openFolderAlias(const QString&)), gui, SLOT(slotFolderOpenAction(QString))); - connect( ProgressDispatcher::instance(), SIGNAL(progressInfo(QString, Progress::Info)), - _accountSettings, SLOT(slotSetProgress(QString, Progress::Info)) ); + connect( ProgressDispatcher::instance(), SIGNAL(progressInfo(QString, ProgressInfo)), + _accountSettings, SLOT(slotSetProgress(QString, ProgressInfo)) ); // default to Account diff --git a/src/libsync/progressdispatcher.cpp b/src/libsync/progressdispatcher.cpp index dc6b9d962..caaf8e032 100644 --- a/src/libsync/progressdispatcher.cpp +++ b/src/libsync/progressdispatcher.cpp @@ -113,7 +113,7 @@ ProgressDispatcher::~ProgressDispatcher() } -void ProgressDispatcher::setProgressInfo(const QString& folder, const Progress::Info& progress) +void ProgressDispatcher::setProgressInfo(const QString& folder, const ProgressInfo& progress) { if( folder.isEmpty()) // The update phase now also has progress @@ -125,5 +125,194 @@ void ProgressDispatcher::setProgressInfo(const QString& folder, const Progress:: emit progressInfo( folder, progress ); } +void ProgressInfo::start() +{ + connect(&_updateEstimatesTimer, SIGNAL(timeout()), SLOT(updateEstimates())); + _updateEstimatesTimer.start(1000); +} + +bool ProgressInfo::hasStarted() const +{ + return _updateEstimatesTimer.isActive(); +} + +void ProgressInfo::adjustTotalsForFile(const SyncFileItem &item) +{ + if (!item._isDirectory) { + _fileProgress._total++; + if (isSizeDependent(item)) { + _sizeProgress._total += item._size; + } + } else if (item._instruction != CSYNC_INSTRUCTION_NONE) { + // Added or removed directories certainly count. + _fileProgress._total++; + } +} + +void ProgressInfo::adjustTotalSize(qint64 change) +{ + _sizeProgress._total += change; +} + +quint64 ProgressInfo::totalFiles() const +{ + return _fileProgress._total; +} + +quint64 ProgressInfo::completedFiles() const +{ + return _fileProgress._completed; +} + +quint64 ProgressInfo::currentFile() const +{ + return completedFiles() + _currentItems.size(); +} + +quint64 ProgressInfo::totalSize() const +{ + return _sizeProgress._total; +} + +quint64 ProgressInfo::completedSize() const +{ + return _sizeProgress._completed; +} + +void ProgressInfo::setProgressComplete(const SyncFileItem &item) +{ + _currentItems.remove(item._file); + _fileProgress._completed += item._affectedItems; + if (ProgressInfo::isSizeDependent(item)) { + _totalSizeOfCompletedJobs += item._size; + } + recomputeCompletedSize(); + _lastCompletedItem = item; +} + +void ProgressInfo::setProgressItem(const SyncFileItem &item, quint64 size) +{ + _currentItems[item._file]._item = item; + _currentItems[item._file]._progress._completed = size; + _currentItems[item._file]._progress._total = item._size; + recomputeCompletedSize(); + + // This seems dubious! + _lastCompletedItem = SyncFileItem(); +} + +ProgressInfo::Estimates ProgressInfo::totalProgress() const +{ + Estimates file = _fileProgress.estimates(); + if (_sizeProgress._total == 0) { + return file; + } + + Estimates size = _sizeProgress.estimates(); + + // Ideally the remaining time would be modeled as: + // remaning_file_sizes / transfer_speed + // + remaining_file_count * per_file_overhead + // + remaining_chunked_file_sizes / chunked_reassembly_speed + // with us estimating the three parameters in conjunction. + // + // But we currently only model the bandwidth and the files per + // second independently, which leads to incorrect values. To slightly + // mitigate this problem, we combine the two models depending on + // which factor dominates (essentially big-file-upload vs. + // many-small-files) + // + // If we have size information, we prefer an estimate based + // on the upload speed. That's particularly relevant for large file + // up/downloads, where files per second will be close to 0. + // + // However, when many *small* files are transfered, the estimate + // can become very pessimistic as the transfered amount per second + // drops significantly. + // + // So, if we detect a high rate of files per second, we gradually prefer + // a file-per-second estimate and assume the remaining transfer will + // be done with the highest speed we've seen. + quint64 combinedEta = file.estimatedEta + _sizeProgress.remaining() / _maxBytesPerSecond * 1000; + if (combinedEta < size.estimatedEta) { + double filesPerSec = _fileProgress._progressPerSec; + // value between 0 (fps==5) and 1 (fps==20) + double scale = qBound(0.0, (filesPerSec - 5.0) / 15.0, 1.0); + size.estimatedEta = (1.0 - scale) * size.estimatedEta + scale * combinedEta; + } + return size; +} + +ProgressInfo::Estimates ProgressInfo::fileProgress(const SyncFileItem &item) const +{ + return _currentItems[item._file]._progress.estimates(); +} + +void ProgressInfo::updateEstimates() +{ + _sizeProgress.update(); + _fileProgress.update(); + + // Update progress of all running items. + QMutableHashIterator it(_currentItems); + while (it.hasNext()) { + it.next(); + it.value()._progress.update(); + } + + _maxFilesPerSecond = qMax(_fileProgress._progressPerSec, + _maxFilesPerSecond); + _maxBytesPerSecond = qMax(_sizeProgress._progressPerSec, + _maxBytesPerSecond); +} + +void ProgressInfo::recomputeCompletedSize() +{ + quint64 r = _totalSizeOfCompletedJobs; + foreach(const ProgressItem &i, _currentItems) { + if (isSizeDependent(i._item)) + r += i._progress._completed; + } + _sizeProgress._completed = r; +} + +ProgressInfo::Estimates ProgressInfo::Progress::estimates() const +{ + Estimates est; + est.estimatedBandwidth = _progressPerSec; + if (_progressPerSec != 0) { + est.estimatedEta = (_total - _completed) / _progressPerSec * 1000.0; + } else { + est.estimatedEta = 0; // looks better than quint64 max + } + return est; +} + +quint64 ProgressInfo::Progress::completed() const +{ + return _completed; +} + +quint64 ProgressInfo::Progress::remaining() const +{ + return _total - _completed; +} + +void ProgressInfo::Progress::update() +{ + // A good way to think about the smoothing factor: + // If we make progress P per sec and then stop making progress at all, + // after N calls to this function (and thus seconds) the _progressPerSec + // will have reduced to P*smoothing^N. + // With a value of 0.9, only 4% of the original value is left after 30s + // + // In the first few updates we want to go to the correct value quickly. + // Therefore, smoothing starts at 0 and ramps up to its final value over time. + const double smoothing = 0.9 * (1.0 - _initialSmoothing); + _initialSmoothing *= 0.7; // goes from 1 to 0.03 in 10s + _progressPerSec = smoothing * _progressPerSec + (1.0 - smoothing) * (_completed - _prevCompleted); + _prevCompleted = _completed; +} + } diff --git a/src/libsync/progressdispatcher.h b/src/libsync/progressdispatcher.h index f4e657b35..42ae64e5b 100644 --- a/src/libsync/progressdispatcher.h +++ b/src/libsync/progressdispatcher.h @@ -20,17 +20,60 @@ #include #include #include +#include #include #include "syncfileitem.h" namespace OCC { -/** - * @brief The FolderScheduler class schedules folders for sync - */ -namespace Progress +class ProgressInfo : public QObject { + Q_OBJECT +public: + ProgressInfo() + : _totalSizeOfCompletedJobs(0) + , _maxFilesPerSecond(2.0) + , _maxBytesPerSecond(100000.0) + {} + + /** + * Called when propagation starts. + * + * hasStarted() will return true afterwards. + */ + void start(); + + /** + * Returns true when propagation has started (start() was called). + * + * This is used when the SyncEngine wants to indicate a new sync + * is about to start via the transmissionProgress() signal. The + * first ProgressInfo will have hasStarted() == false. + */ + bool hasStarted() const; + + /** + * Increase the file and size totals by the amount indicated in item. + */ + void adjustTotalsForFile(const SyncFileItem & item); + + /** + * Adjust the total size by some value. + * + * Deprecated. Used only in the legacy propagator. + */ + void adjustTotalSize(qint64 change); + + quint64 totalFiles() const; + quint64 completedFiles() const; + + quint64 totalSize() const; + quint64 completedSize() const; + + /** Number of a file that is currently in progress. */ + quint64 currentFile() const; + /** Return true is the size need to be taken in account in the total amount of time */ static inline bool isSizeDependent(const SyncFileItem & item) { @@ -40,137 +83,111 @@ namespace Progress || item._instruction == CSYNC_INSTRUCTION_NEW); } + /** + * Holds estimates about progress, returned to the user. + */ + struct Estimates + { + /// Estimated completion amount per second. (of bytes or files) + quint64 estimatedBandwidth; - struct Info { - Info() : _totalFileCount(0), _totalSize(0), _completedFileCount(0), _completedSize(0) {} - - // Used during local and remote update phase - QString _currentDiscoveredFolder; - - quint64 _totalFileCount; - quint64 _totalSize; - quint64 _completedFileCount; - quint64 _completedSize; - // Should this be in a separate file? - struct EtaEstimate { - EtaEstimate() - : _startedTime(QDateTime::currentMSecsSinceEpoch()) - , _agvEtaMSecs(0) - , _effectivProgressPerSec(0) - , _sampleCount(1) - { - } - - static const int MAX_AVG_DIVIDER=60; - static const int INITAL_WAIT_TIME=5; - - quint64 _startedTime ; - quint64 _agvEtaMSecs; - quint64 _effectivProgressPerSec; - float _sampleCount; - - /** - * reset the estiamte. - */ - void reset() { - _startedTime = QDateTime::currentMSecsSinceEpoch(); - _sampleCount =1; - _effectivProgressPerSec = _agvEtaMSecs = 0; - } - - /** - * update the estimated eta time with more current data. - * @param quint64 completed the amount the was completed. - * @param quint64 total the total amout that should be completed. - */ - void updateTime(quint64 completed, quint64 total) { - quint64 elapsedTime = QDateTime::currentMSecsSinceEpoch() - this->_startedTime ; - //don't start until you have some good data to process, prevents jittring estiamtes at the start of the syncing process - if(total != 0 && completed != 0 && elapsedTime > INITAL_WAIT_TIME ) { - if(_sampleCount < MAX_AVG_DIVIDER) { _sampleCount+=0.01f; } - // (elapsedTime-1) is an hack to avoid float "rounding" issue (ie. 0.99999999999999999999....) - _agvEtaMSecs = _agvEtaMSecs + (((static_cast(total) / completed) * elapsedTime) - (elapsedTime-1)) - this->getEtaEstimate(); - _effectivProgressPerSec = ( total - completed ) / (1+this->getEtaEstimate()/1000); - } - } - - /** - * Get the eta estimate in milliseconds - * @return quint64 the estimate amount of milliseconds to end the process. - */ - quint64 getEtaEstimate() const { - return _agvEtaMSecs / _sampleCount; - } - - /** - * Get the estimated average bandwidth usage. - * @return quint64 the estimated bandwidth usage in bytes. - */ - quint64 getEstimatedBandwidth() const { - return _effectivProgressPerSec; - } - }; - EtaEstimate _totalEtaEstimate; - - struct ProgressItem { - ProgressItem() : _completedSize(0) {} - SyncFileItem _item; - quint64 _completedSize; - EtaEstimate _etaEstimate; - }; - QHash _currentItems; - SyncFileItem _lastCompletedItem; - - void setProgressComplete(const SyncFileItem &item) { - _currentItems.remove(item._file); - _completedFileCount += item._affectedItems; - if (Progress::isSizeDependent(item)) { - _completedSize += item._size; - } - _lastCompletedItem = item; - this->updateEstimation(); - } - void setProgressItem(const SyncFileItem &item, quint64 size) { - _currentItems[item._file]._item = item; - _currentItems[item._file]._completedSize = size; - _lastCompletedItem = SyncFileItem(); - this->updateEstimation(); - _currentItems[item._file]._etaEstimate.updateTime(size,item._size); - } - - void updateEstimation() { - if(this->_totalSize > 0) { - _totalEtaEstimate.updateTime(this->completedSize(),this->_totalSize); - } else { - _totalEtaEstimate.updateTime(this->_completedFileCount,this->_totalFileCount); - } - } - - quint64 completedSize() const { - quint64 r = _completedSize; - foreach(const ProgressItem &i, _currentItems) { - if (Progress::isSizeDependent(i._item)) - r += i._completedSize; - } - return r; - } - /** - * Get the total completion estimate structure - * @return EtaEstimate a structure containing the total completion information. - */ - EtaEstimate totalEstimate() const { - return _totalEtaEstimate; - } - - /** - * Get the current file completion estimate structure - * @return EtaEstimate a structure containing the current file completion information. - */ - EtaEstimate getFileEstimate(const SyncFileItem &item) const { - return _currentItems[item._file]._etaEstimate; - } + /// Estimated eta in milliseconds. + quint64 estimatedEta; }; + /** + * Holds the current state of something making progress and maintains an + * estimate of the current progress per second. + */ + struct Progress + { + Progress() + : _progressPerSec(0) + , _completed(0) + , _prevCompleted(0) + , _total(0) + , _initialSmoothing(1.0) + { + } + + /** Returns the estimates about progress per second and eta. */ + Estimates estimates() const; + + quint64 completed() const; + quint64 remaining() const; + + private: + /** + * Update the exponential moving average estimate of _progressPerSec. + */ + void update(); + + double _progressPerSec; + quint64 _completed; + quint64 _prevCompleted; + quint64 _total; + + // Used to get to a good value faster when + // progress measurement stats. See update(). + double _initialSmoothing; + + friend class ProgressInfo; + }; + + struct ProgressItem + { + SyncFileItem _item; + Progress _progress; + }; + QHash _currentItems; + + SyncFileItem _lastCompletedItem; + + // Used during local and remote update phase + QString _currentDiscoveredFolder; + + void setProgressComplete(const SyncFileItem &item); + + void setProgressItem(const SyncFileItem &item, quint64 size); + + /** + * Get the total completion estimate + */ + Estimates totalProgress() const; + + /** + * Get the current file completion estimate structure + */ + Estimates fileProgress(const SyncFileItem &item) const; + +private slots: + /** + * Called every second once started, this function updates the + * estimates. + */ + void updateEstimates(); + +private: + // Sets the completed size by summing finished jobs with the progress + // of active ones. + void recomputeCompletedSize(); + + // Triggers the update() slot every second once propagation started. + QTimer _updateEstimatesTimer; + + Progress _sizeProgress; + Progress _fileProgress; + + // All size from completed jobs only. + quint64 _totalSizeOfCompletedJobs; + + // The fastest observed rate of files per second in this sync. + double _maxFilesPerSecond; + double _maxBytesPerSecond; +}; + +namespace Progress { + OWNCLOUDSYNC_EXPORT QString asActionString( const SyncFileItem& item ); OWNCLOUDSYNC_EXPORT QString asResultString( const SyncFileItem& item ); @@ -205,7 +222,7 @@ signals: @param[out] progress A struct with all progress info. */ - void progressInfo( const QString& folder, const Progress::Info& progress ); + void progressInfo( const QString& folder, const ProgressInfo& progress ); /** * @brief: the item's job is completed */ @@ -214,7 +231,7 @@ signals: void syncItemDiscovered(const QString &folder, const SyncFileItem & item); protected: - void setProgressInfo(const QString& folder, const Progress::Info& progress); + void setProgressInfo(const QString& folder, const ProgressInfo& progress); private: ProgressDispatcher(QObject* parent = 0); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index ee155765d..633319fe4 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -65,6 +65,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath, , _remoteUrl(remoteURL) , _remotePath(remotePath) , _journal(journal) + , _progressInfo(new ProgressInfo) , _hasNoneFiles(false) , _hasRemoveFile(false) , _uploadLimit(0) @@ -73,7 +74,6 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath, { qRegisterMetaType("SyncFileItem"); qRegisterMetaType("SyncFileItem::Status"); - qRegisterMetaType("Progress::Info"); _thread.setObjectName("CSync_Neon_Thread"); _thread.start(); @@ -494,15 +494,8 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) checkErrorBlacklisting( *item ); } - if (!item->_isDirectory) { - _progressInfo._totalFileCount++; - if (Progress::isSizeDependent(item)) { - _progressInfo._totalSize += file->size; - } - } else if (file->instruction != CSYNC_INSTRUCTION_NONE) { - // Added or removed directories certainly count. - _progressInfo._totalFileCount++; - } + _progressInfo->adjustTotalsForFile(*item); + _needsUpdate = true; item->log._etag = file->etag; @@ -702,8 +695,6 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) qDebug() << "<<#### Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Reconcile Finished")); - _progressInfo = Progress::Info(); - _hasNoneFiles = false; _hasRemoveFile = false; bool walkOk = true; @@ -744,9 +735,9 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) // To announce the beginning of the sync emit aboutToPropagate(_syncedItems); - _progressInfo._completedFileCount = ULLONG_MAX; // indicate the start with max - emit transmissionProgress(_progressInfo); - _progressInfo._completedFileCount = 0; + // it's important to do this before ProgressInfo::start(), to announce start of new sync + emit transmissionProgress(*_progressInfo); + _progressInfo->start(); if (!_hasNoneFiles && _hasRemoveFile) { qDebug() << Q_FUNC_INFO << "All the files are going to be changed, asking the user"; @@ -843,13 +834,13 @@ void SyncEngine::slotJobCompleted(const SyncFileItem &item) const char * instruction_str = csync_instruction_str(item._instruction); qDebug() << Q_FUNC_INFO << item._file << instruction_str << item._status << item._errorString; - _progressInfo.setProgressComplete(item); + _progressInfo->setProgressComplete(item); if (item._status == SyncFileItem::FatalError) { emit csyncError(item._errorString); } - emit transmissionProgress(_progressInfo); + emit transmissionProgress(*_progressInfo); emit jobCompleted(item); } @@ -891,14 +882,14 @@ void SyncEngine::finalize() void SyncEngine::slotProgress(const SyncFileItem& item, quint64 current) { - _progressInfo.setProgressItem(item, current); - emit transmissionProgress(_progressInfo); + _progressInfo->setProgressItem(item, current); + emit transmissionProgress(*_progressInfo); } void SyncEngine::slotAdjustTotalTransmissionSize(qint64 change) { - _progressInfo._totalSize += change; + _progressInfo->adjustTotalSize(change); } /* Given a path on the remote, give the path as it is when the rename is done */ diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index bd740663e..baa3ffcfd 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -99,7 +99,7 @@ signals: // after sync is done void treeWalkResult(const SyncFileItemVector&); - void transmissionProgress( const Progress::Info& progress ); + void transmissionProgress( const ProgressInfo& progress ); void csyncStateDbFile( const QString& ); void wipeDb(); @@ -176,7 +176,7 @@ private: QThread _thread; - Progress::Info _progressInfo; + QScopedPointer _progressInfo; Utility::StopWatch _stopWatch;