mirror of
https://github.com/nextcloud/desktop.git
synced 2024-10-27 23:17:13 +03:00
SyncJournalDb: Reimplement the db_is_empty logic from csync
This reduces the initial sync local discovery time from 2.0 to 0.6 seconds in LargeSyncBench on my machine.
This commit is contained in:
parent
185cb2e39a
commit
ecb4e96794
5 changed files with 24 additions and 28 deletions
|
@ -76,7 +76,9 @@ static QString defaultJournalMode(const QString &dbPath)
|
|||
SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent)
|
||||
: QObject(parent)
|
||||
, _dbFile(dbFilePath)
|
||||
, _mutex(QMutex::Recursive)
|
||||
, _transaction(0)
|
||||
, _metadataTableIsEmpty(false)
|
||||
{
|
||||
// Allow forcing the journal mode for debugging
|
||||
static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE"));
|
||||
|
@ -672,6 +674,10 @@ bool SyncJournalDb::checkConnect()
|
|||
// don't start a new transaction now
|
||||
commitInternal(QString("checkConnect End"), false);
|
||||
|
||||
// This avoid reading from the DB if we already know it is empty
|
||||
// thereby speeding up the initial discovery significantly.
|
||||
_metadataTableIsEmpty = (getFileRecordCount() == 0);
|
||||
|
||||
// Hide 'em all!
|
||||
FileSystem::setFileHidden(databaseFilePath(), true);
|
||||
FileSystem::setFileHidden(databaseFilePath() + "-wal", true);
|
||||
|
@ -715,6 +721,7 @@ void SyncJournalDb::close()
|
|||
|
||||
_db.close();
|
||||
_avoidReadFromDbOnNextSyncFilter.clear();
|
||||
_metadataTableIsEmpty = false;
|
||||
}
|
||||
|
||||
|
||||
|
@ -973,6 +980,9 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
|
|||
return false;
|
||||
}
|
||||
|
||||
// Can't be true anymore.
|
||||
_metadataTableIsEmpty = false;
|
||||
|
||||
return true;
|
||||
} else {
|
||||
qCWarning(lcDb) << "Failed to connect database.";
|
||||
|
@ -1020,6 +1030,9 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
|||
rec->_path.clear();
|
||||
Q_ASSERT(!rec->isValid());
|
||||
|
||||
if (_metadataTableIsEmpty)
|
||||
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||
|
||||
if (!checkConnect())
|
||||
return false;
|
||||
|
||||
|
@ -1028,7 +1041,6 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
|||
_getFileRecordQuery->bindValue(1, getPHash(filename));
|
||||
|
||||
if (!_getFileRecordQuery->exec()) {
|
||||
locker.unlock();
|
||||
close();
|
||||
return false;
|
||||
}
|
||||
|
@ -1040,9 +1052,7 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
|||
if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE
|
||||
QString err = _getFileRecordQuery->error();
|
||||
qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err;
|
||||
locker.unlock();
|
||||
close();
|
||||
locker.relock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1058,7 +1068,7 @@ bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *r
|
|||
rec->_path.clear();
|
||||
Q_ASSERT(!rec->isValid());
|
||||
|
||||
if (!inode)
|
||||
if (!inode || _metadataTableIsEmpty)
|
||||
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||
|
||||
if (!checkConnect())
|
||||
|
@ -1087,7 +1097,7 @@ bool SyncJournalDb::getFileRecordByFileId(const QByteArray &fileId, SyncJournalF
|
|||
rec->_path.clear();
|
||||
Q_ASSERT(!rec->isValid());
|
||||
|
||||
if (fileId.isEmpty())
|
||||
if (fileId.isEmpty() || _metadataTableIsEmpty)
|
||||
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||
|
||||
if (!checkConnect())
|
||||
|
@ -1111,6 +1121,9 @@ bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::functio
|
|||
{
|
||||
QMutexLocker locker(&_mutex);
|
||||
|
||||
if (_metadataTableIsEmpty)
|
||||
return true; // no error, yet nothing found
|
||||
|
||||
if (!checkConnect())
|
||||
return false;
|
||||
|
||||
|
@ -1184,15 +1197,11 @@ int SyncJournalDb::getFileRecordCount()
|
|||
{
|
||||
QMutexLocker locker(&_mutex);
|
||||
|
||||
if (!checkConnect()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
SqlQuery query(_db);
|
||||
query.prepare("SELECT COUNT(*) FROM metadata");
|
||||
|
||||
if (!query.exec()) {
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (query.next()) {
|
||||
|
@ -1200,7 +1209,7 @@ int SyncJournalDb::getFileRecordCount()
|
|||
return count;
|
||||
}
|
||||
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
|
||||
bool SyncJournalDb::updateFileRecordChecksum(const QString &filename,
|
||||
|
@ -1775,7 +1784,6 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path)
|
|||
|
||||
// We also need to remove the ETags so the update phase refreshes the directory paths
|
||||
// on the next sync
|
||||
locker.unlock();
|
||||
avoidReadFromDbOnNextSync(path);
|
||||
}
|
||||
|
||||
|
|
|
@ -66,7 +66,6 @@ public:
|
|||
bool setFileRecordMetadata(const SyncJournalFileRecord &record);
|
||||
|
||||
bool deleteFileRecord(const QString &filename, bool recursively = false);
|
||||
int getFileRecordCount();
|
||||
bool updateFileRecordChecksum(const QString &filename,
|
||||
const QByteArray &contentChecksum,
|
||||
const QByteArray &contentChecksumType);
|
||||
|
@ -215,6 +214,7 @@ public:
|
|||
void clearFileTable();
|
||||
|
||||
private:
|
||||
int getFileRecordCount();
|
||||
bool updateDatabaseStructure();
|
||||
bool updateMetadataTableStructure();
|
||||
bool updateErrorBlacklistTableStructure();
|
||||
|
@ -237,6 +237,7 @@ private:
|
|||
QString _dbFile;
|
||||
QMutex _mutex; // Public functions are protected with the mutex.
|
||||
int _transaction;
|
||||
bool _metadataTableIsEmpty;
|
||||
|
||||
// NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close
|
||||
QScopedPointer<SqlQuery> _getFileRecordQuery;
|
||||
|
|
|
@ -310,7 +310,6 @@ int csync_s::reinitialize() {
|
|||
|
||||
remote.read_from_db = 0;
|
||||
read_remote_from_db = true;
|
||||
db_is_empty = false;
|
||||
|
||||
local.files.clear();
|
||||
remote.files.clear();
|
||||
|
|
|
@ -142,12 +142,6 @@ struct OCSYNC_EXPORT csync_s {
|
|||
*/
|
||||
bool read_remote_from_db = false;
|
||||
|
||||
/**
|
||||
* If true, the DB is considered empty and all reads are skipped. (default is false)
|
||||
* This is useful during the initial local discovery as it speeds it up significantly.
|
||||
*/
|
||||
bool db_is_empty = false;
|
||||
|
||||
bool ignore_hidden_files = true;
|
||||
|
||||
csync_s(const char *localUri, OCC::SyncJournalDb *statedb);
|
||||
|
|
|
@ -786,7 +786,6 @@ void SyncEngine::startSync()
|
|||
|
||||
csync_resume(_csync_ctx.data());
|
||||
|
||||
int fileRecordCount = -1;
|
||||
if (!_journal->exists()) {
|
||||
qCInfo(lcEngine) << "New sync (no sync journal exists)";
|
||||
} else {
|
||||
|
@ -800,9 +799,8 @@ void SyncEngine::startSync()
|
|||
verStr.append(" on ").append(Utility::platformName());
|
||||
qCInfo(lcEngine) << verStr;
|
||||
|
||||
fileRecordCount = _journal->getFileRecordCount(); // this creates the DB if it does not exist yet
|
||||
|
||||
if (fileRecordCount == -1) {
|
||||
// This creates the DB if it does not exist yet.
|
||||
if (!_journal->isConnected()) {
|
||||
qCWarning(lcEngine) << "No way to create a sync journal!";
|
||||
csyncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder."));
|
||||
finalize(false);
|
||||
|
@ -812,10 +810,6 @@ void SyncEngine::startSync()
|
|||
|
||||
_csync_ctx->read_remote_from_db = true;
|
||||
|
||||
// This tells csync to never read from the DB if it is empty
|
||||
// thereby speeding up the initial discovery significantly.
|
||||
_csync_ctx->db_is_empty = (fileRecordCount == 0);
|
||||
|
||||
bool ok;
|
||||
auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok);
|
||||
if (ok) {
|
||||
|
|
Loading…
Reference in a new issue