Add a key value store to the sync journal

This key value store should help to keep track of important events,
that can not be store in the logs, because the logs are deleted too fast.

Signed-off-by: Felix Weilbach <felix.weilbach@nextcloud.com>

src/common/syncjournaldb.cpp

@@ -408,6 +408,12 @@ bool SyncJournalDb::checkConnect()
         return sqlFail(QStringLiteral("Create table metadata"), createQuery);
     }
 
+    createQuery.prepare("CREATE TABLE IF NOT EXISTS key_value_store(key VARCHAR(4096), value VARCHAR(4096), PRIMARY KEY(key));");
+
+    if (!createQuery.exec()) {
+        return sqlFail(QStringLiteral("Create table key_value_store"), createQuery);
+    }
+
     createQuery.prepare("CREATE TABLE IF NOT EXISTS downloadinfo("
                         "path VARCHAR(4096),"
                         "tmpfile VARCHAR(4096),"
@@ -970,6 +976,43 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
     }
 }
 
+void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect()) {
+        return;
+    }
+
+    if (!_setKeyValueStoreQuery.initOrReset(QByteArrayLiteral("INSERT OR REPLACE INTO key_value_store (key, value) VALUES(?1, ?2);"), _db)) {
+        return;
+    }
+
+    _setKeyValueStoreQuery.bindValue(1, key);
+    _setKeyValueStoreQuery.bindValue(2, QString::number(value));
+    _setKeyValueStoreQuery.exec();
+}
+
+qint64 SyncJournalDb::keyValueStoreGetInt(const QString &key, qint64 defaultValue)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect()) {
+        return defaultValue;
+    }
+
+    if (!_getKeyValueStoreQuery.initOrReset(QByteArrayLiteral("SELECT value FROM key_value_store WHERE key = ?1;"), _db)) {
+        return defaultValue;
+    }
+
+    _getKeyValueStoreQuery.bindValue(1, key);
+    _getKeyValueStoreQuery.exec();
+
+    if (!_getKeyValueStoreQuery.next().hasData) {
+        return defaultValue;
+    }
+
+    return _getKeyValueStoreQuery.int64Value(0);
+}
+
 // TODO: filename -> QBytearray?
 bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively)
 {

src/common/syncjournaldb.h

@@ -20,9 +20,9 @@
 #define SYNCJOURNALDB_H
 
 #include <QObject>
-#include <qmutex.h>
 #include <QDateTime>
 #include <QHash>
+#include <QMutex>
 #include <functional>
 
 #include "common/utility.h"
@@ -66,6 +66,9 @@ public:
     bool listFilesInPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback);
     bool setFileRecord(const SyncJournalFileRecord &record);
 
+    void keyValueStoreSet(const QString &key, qint64 value);
+    qint64 keyValueStoreGetInt(const QString &key, qint64 defaultValue);
+
     bool deleteFileRecord(const QString &filename, bool recursively = false);
     bool updateFileRecordChecksum(const QString &filename,
         const QByteArray &contentChecksum,
@@ -418,6 +421,8 @@ private:
     SqlQuery _getDataFingerprintQuery;
     SqlQuery _setDataFingerprintQuery1;
     SqlQuery _setDataFingerprintQuery2;
+    SqlQuery _setKeyValueStoreQuery;
+    SqlQuery _getKeyValueStoreQuery;
     SqlQuery _getConflictRecordQuery;
     SqlQuery _setConflictRecordQuery;
     SqlQuery _deleteConflictRecordQuery;

src/libsync/discovery.cpp

@@ -329,6 +329,13 @@ void ProcessDirectoryJob::processFile(PathTuple path,
                               << " | e2ee: " << dbEntry._isE2eEncrypted << "/" << serverEntry.isE2eEncrypted
                               << " | e2eeMangledName: " << dbEntry.e2eMangledName() << "/" << serverEntry.e2eMangledName;
 
+    if (localEntry.isValid()
+        && !serverEntry.isValid()
+        && !dbEntry.isValid()
+        && localEntry.modtime < _lastSyncTimestamp) {
+        qCWarning(lcDisco) << "File" << path._original << "was modified before the last sync run and is not in the sync journal and server";
+    }
+
     if (_discoveryData->isRenamed(path._original)) {
         qCDebug(lcDisco) << "Ignoring renamed";
         return; // Ignore this.
@@ -1185,7 +1192,8 @@ void ProcessDirectoryJob::processFileFinalize(
         recurse = false;
     }
     if (recurse) {
-        auto job = new ProcessDirectoryJob(path, item, recurseQueryLocal, recurseQueryServer, this);
+        auto job = new ProcessDirectoryJob(path, item, recurseQueryLocal, recurseQueryServer,
+            _lastSyncTimestamp, this);
         job->setInsideEncryptedTree(isInsideEncryptedTree() || item->_isEncrypted);
         if (removed) {
             job->setParent(_discoveryData);
@@ -1228,7 +1236,7 @@ void ProcessDirectoryJob::processBlacklisted(const PathTuple &path, const OCC::L
     qCInfo(lcDisco) << "Discovered (blacklisted) " << item->_file << item->_instruction << item->_direction << item->isDirectory();
 
     if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) {
-        auto job = new ProcessDirectoryJob(path, item, NormalQuery, InBlackList, this);
+        auto job = new ProcessDirectoryJob(path, item, NormalQuery, InBlackList, _lastSyncTimestamp, this);
         connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
         _queuedJobs.push_back(job);
     } else {

src/libsync/discovery.h

@@ -63,8 +63,10 @@ public:
      *
      * The base pin state is used if the root dir's pin state can't be retrieved.
      */
-    explicit ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState, QObject *parent)
+    explicit ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState,
+        qint64 lastSyncTimestamp, QObject *parent)
         : QObject(parent)
+        , _lastSyncTimestamp(lastSyncTimestamp)
         , _discoveryData(data)
     {
         computePinState(basePinState);
@@ -72,7 +74,7 @@ public:
 
     /// For creating subjobs
     explicit ProcessDirectoryJob(const PathTuple &path, const SyncFileItemPtr &dirItem,
-        QueryMode queryLocal, QueryMode queryServer,
+        QueryMode queryLocal, QueryMode queryServer, qint64 lastSyncTimestamp,
         ProcessDirectoryJob *parent)
         : QObject(parent)
         , _dirItem(dirItem)
@@ -80,6 +82,7 @@ public:
         , _queryLocal(queryLocal)
         , _discoveryData(parent->_discoveryData)
         , _currentFolder(path)
+        , _lastSyncTimestamp(lastSyncTimestamp)
     {
         computePinState(parent->_pinState);
     }
@@ -242,6 +245,8 @@ private:
      */
     void setupDbPinStateActions(SyncJournalFileRecord &record);
 
+    qint64 _lastSyncTimestamp = 0;
+
     QueryMode _queryServer = QueryMode::NormalQuery;
     QueryMode _queryLocal = QueryMode::NormalQuery;
 

src/libsync/syncengine.cpp

@@ -111,6 +111,9 @@ SyncEngine::SyncEngine(AccountPtr account, const QString &localPath,
     _clearTouchedFilesTimer.setSingleShot(true);
     _clearTouchedFilesTimer.setInterval(30 * 1000);
     connect(&_clearTouchedFilesTimer, &QTimer::timeout, this, &SyncEngine::slotClearTouchedFiles);
+    connect(this, &SyncEngine::finished, [this](bool /* finished */) {
+        _journal->keyValueStoreSet("last_sync", QDateTime::currentSecsSinceEpoch());
+    });
 }
 
 SyncEngine::~SyncEngine()
@@ -586,7 +589,7 @@ void SyncEngine::startSync()
         _syncFileStatusTracker.data(), &SyncFileStatusTracker::slotAddSilentlyExcluded);
 
     auto discoveryJob = new ProcessDirectoryJob(
-        _discoveryPhase.data(), PinState::AlwaysLocal, _discoveryPhase.data());
+        _discoveryPhase.data(), PinState::AlwaysLocal, _journal->keyValueStoreGetInt("last_sync", 0), _discoveryPhase.data());
     _discoveryPhase->startJob(discoveryJob);
     connect(discoveryJob, &ProcessDirectoryJob::etag, this, &SyncEngine::slotRootEtagReceived);
 }