Merge pull request #3257 from nextcloud/feature/store-last-sync-run

Track time of last sync

src/common/syncjournaldb.cpp

@@ -408,6 +408,12 @@ bool SyncJournalDb::checkConnect()
         return sqlFail(QStringLiteral("Create table metadata"), createQuery);
     }
 
+    createQuery.prepare("CREATE TABLE IF NOT EXISTS key_value_store(key VARCHAR(4096), value VARCHAR(4096), PRIMARY KEY(key));");
+
+    if (!createQuery.exec()) {
+        return sqlFail(QStringLiteral("Create table key_value_store"), createQuery);
+    }
+
     createQuery.prepare("CREATE TABLE IF NOT EXISTS downloadinfo("
                         "path VARCHAR(4096),"
                         "tmpfile VARCHAR(4096),"
@@ -970,6 +976,43 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
     }
 }
 
+void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect()) {
+        return;
+    }
+
+    if (!_setKeyValueStoreQuery.initOrReset(QByteArrayLiteral("INSERT OR REPLACE INTO key_value_store (key, value) VALUES(?1, ?2);"), _db)) {
+        return;
+    }
+
+    _setKeyValueStoreQuery.bindValue(1, key);
+    _setKeyValueStoreQuery.bindValue(2, QString::number(value));
+    _setKeyValueStoreQuery.exec();
+}
+
+qint64 SyncJournalDb::keyValueStoreGetInt(const QString &key, qint64 defaultValue)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect()) {
+        return defaultValue;
+    }
+
+    if (!_getKeyValueStoreQuery.initOrReset(QByteArrayLiteral("SELECT value FROM key_value_store WHERE key = ?1;"), _db)) {
+        return defaultValue;
+    }
+
+    _getKeyValueStoreQuery.bindValue(1, key);
+    _getKeyValueStoreQuery.exec();
+
+    if (!_getKeyValueStoreQuery.next().hasData) {
+        return defaultValue;
+    }
+
+    return _getKeyValueStoreQuery.int64Value(0);
+}
+
 // TODO: filename -> QBytearray?
 bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively)
 {

src/common/syncjournaldb.h

@@ -20,9 +20,9 @@
 #define SYNCJOURNALDB_H
 
 #include <QObject>
-#include <qmutex.h>
 #include <QDateTime>
 #include <QHash>
+#include <QMutex>
 #include <functional>
 
 #include "common/utility.h"
@@ -66,6 +66,9 @@ public:
     bool listFilesInPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback);
     bool setFileRecord(const SyncJournalFileRecord &record);
 
+    void keyValueStoreSet(const QString &key, qint64 value);
+    qint64 keyValueStoreGetInt(const QString &key, qint64 defaultValue);
+
     bool deleteFileRecord(const QString &filename, bool recursively = false);
     bool updateFileRecordChecksum(const QString &filename,
         const QByteArray &contentChecksum,
@@ -418,6 +421,8 @@ private:
     SqlQuery _getDataFingerprintQuery;
     SqlQuery _setDataFingerprintQuery1;
     SqlQuery _setDataFingerprintQuery2;
+    SqlQuery _setKeyValueStoreQuery;
+    SqlQuery _getKeyValueStoreQuery;
     SqlQuery _getConflictRecordQuery;
     SqlQuery _setConflictRecordQuery;
     SqlQuery _deleteConflictRecordQuery;

src/libsync/discovery.cpp

@@ -329,6 +329,13 @@ void ProcessDirectoryJob::processFile(PathTuple path,
                               << " | e2ee: " << dbEntry._isE2eEncrypted << "/" << serverEntry.isE2eEncrypted
                               << " | e2eeMangledName: " << dbEntry.e2eMangledName() << "/" << serverEntry.e2eMangledName;
 
+    if (localEntry.isValid()
+        && !serverEntry.isValid()
+        && !dbEntry.isValid()
+        && localEntry.modtime < _lastSyncTimestamp) {
+        qCWarning(lcDisco) << "File" << path._original << "was modified before the last sync run and is not in the sync journal and server";
+    }
+
     if (_discoveryData->isRenamed(path._original)) {
         qCDebug(lcDisco) << "Ignoring renamed";
         return; // Ignore this.
@@ -1185,7 +1192,8 @@ void ProcessDirectoryJob::processFileFinalize(
         recurse = false;
     }
     if (recurse) {
-        auto job = new ProcessDirectoryJob(path, item, recurseQueryLocal, recurseQueryServer, this);
+        auto job = new ProcessDirectoryJob(path, item, recurseQueryLocal, recurseQueryServer,
+            _lastSyncTimestamp, this);
         job->setInsideEncryptedTree(isInsideEncryptedTree() || item->_isEncrypted);
         if (removed) {
             job->setParent(_discoveryData);
@@ -1228,7 +1236,7 @@ void ProcessDirectoryJob::processBlacklisted(const PathTuple &path, const OCC::L
     qCInfo(lcDisco) << "Discovered (blacklisted) " << item->_file << item->_instruction << item->_direction << item->isDirectory();
 
     if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) {
-        auto job = new ProcessDirectoryJob(path, item, NormalQuery, InBlackList, this);
+        auto job = new ProcessDirectoryJob(path, item, NormalQuery, InBlackList, _lastSyncTimestamp, this);
         connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
         _queuedJobs.push_back(job);
     } else {

src/libsync/discovery.h

@@ -63,8 +63,10 @@ public:
      *
      * The base pin state is used if the root dir's pin state can't be retrieved.
      */
-    explicit ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState, QObject *parent)
+    explicit ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState,
+        qint64 lastSyncTimestamp, QObject *parent)
         : QObject(parent)
+        , _lastSyncTimestamp(lastSyncTimestamp)
         , _discoveryData(data)
     {
         computePinState(basePinState);
@@ -72,7 +74,7 @@ public:
 
     /// For creating subjobs
     explicit ProcessDirectoryJob(const PathTuple &path, const SyncFileItemPtr &dirItem,
-        QueryMode queryLocal, QueryMode queryServer,
+        QueryMode queryLocal, QueryMode queryServer, qint64 lastSyncTimestamp,
         ProcessDirectoryJob *parent)
         : QObject(parent)
         , _dirItem(dirItem)
@@ -80,6 +82,7 @@ public:
         , _queryLocal(queryLocal)
         , _discoveryData(parent->_discoveryData)
         , _currentFolder(path)
+        , _lastSyncTimestamp(lastSyncTimestamp)
     {
         computePinState(parent->_pinState);
     }
@@ -242,6 +245,8 @@ private:
      */
     void setupDbPinStateActions(SyncJournalFileRecord &record);
 
+    qint64 _lastSyncTimestamp = 0;
+
     QueryMode _queryServer = QueryMode::NormalQuery;
     QueryMode _queryLocal = QueryMode::NormalQuery;
 

src/libsync/syncengine.cpp

@@ -111,6 +111,9 @@ SyncEngine::SyncEngine(AccountPtr account, const QString &localPath,
     _clearTouchedFilesTimer.setSingleShot(true);
     _clearTouchedFilesTimer.setInterval(30 * 1000);
     connect(&_clearTouchedFilesTimer, &QTimer::timeout, this, &SyncEngine::slotClearTouchedFiles);
+    connect(this, &SyncEngine::finished, [this](bool /* finished */) {
+        _journal->keyValueStoreSet("last_sync", QDateTime::currentSecsSinceEpoch());
+    });
 }
 
 SyncEngine::~SyncEngine()
@@ -586,7 +589,7 @@ void SyncEngine::startSync()
         _syncFileStatusTracker.data(), &SyncFileStatusTracker::slotAddSilentlyExcluded);
 
     auto discoveryJob = new ProcessDirectoryJob(
-        _discoveryPhase.data(), PinState::AlwaysLocal, _discoveryPhase.data());
+        _discoveryPhase.data(), PinState::AlwaysLocal, _journal->keyValueStoreGetInt("last_sync", 0), _discoveryPhase.data());
     _discoveryPhase->startJob(discoveryJob);
     connect(discoveryJob, &ProcessDirectoryJob::etag, this, &SyncEngine::slotRootEtagReceived);
 }