mirror of
https://github.com/nextcloud/desktop.git
synced 2024-10-27 23:17:13 +03:00
SyncJournalDb: Reimplement the db_is_empty logic from csync
This reduces the initial sync local discovery time from 2.0 to 0.6 seconds in LargeSyncBench on my machine.
This commit is contained in:
parent
185cb2e39a
commit
ecb4e96794
5 changed files with 24 additions and 28 deletions
|
@ -76,7 +76,9 @@ static QString defaultJournalMode(const QString &dbPath)
|
||||||
SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent)
|
SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent)
|
||||||
: QObject(parent)
|
: QObject(parent)
|
||||||
, _dbFile(dbFilePath)
|
, _dbFile(dbFilePath)
|
||||||
|
, _mutex(QMutex::Recursive)
|
||||||
, _transaction(0)
|
, _transaction(0)
|
||||||
|
, _metadataTableIsEmpty(false)
|
||||||
{
|
{
|
||||||
// Allow forcing the journal mode for debugging
|
// Allow forcing the journal mode for debugging
|
||||||
static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE"));
|
static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE"));
|
||||||
|
@ -672,6 +674,10 @@ bool SyncJournalDb::checkConnect()
|
||||||
// don't start a new transaction now
|
// don't start a new transaction now
|
||||||
commitInternal(QString("checkConnect End"), false);
|
commitInternal(QString("checkConnect End"), false);
|
||||||
|
|
||||||
|
// This avoid reading from the DB if we already know it is empty
|
||||||
|
// thereby speeding up the initial discovery significantly.
|
||||||
|
_metadataTableIsEmpty = (getFileRecordCount() == 0);
|
||||||
|
|
||||||
// Hide 'em all!
|
// Hide 'em all!
|
||||||
FileSystem::setFileHidden(databaseFilePath(), true);
|
FileSystem::setFileHidden(databaseFilePath(), true);
|
||||||
FileSystem::setFileHidden(databaseFilePath() + "-wal", true);
|
FileSystem::setFileHidden(databaseFilePath() + "-wal", true);
|
||||||
|
@ -715,6 +721,7 @@ void SyncJournalDb::close()
|
||||||
|
|
||||||
_db.close();
|
_db.close();
|
||||||
_avoidReadFromDbOnNextSyncFilter.clear();
|
_avoidReadFromDbOnNextSyncFilter.clear();
|
||||||
|
_metadataTableIsEmpty = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -973,6 +980,9 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Can't be true anymore.
|
||||||
|
_metadataTableIsEmpty = false;
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
qCWarning(lcDb) << "Failed to connect database.";
|
qCWarning(lcDb) << "Failed to connect database.";
|
||||||
|
@ -1020,6 +1030,9 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
||||||
rec->_path.clear();
|
rec->_path.clear();
|
||||||
Q_ASSERT(!rec->isValid());
|
Q_ASSERT(!rec->isValid());
|
||||||
|
|
||||||
|
if (_metadataTableIsEmpty)
|
||||||
|
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||||
|
|
||||||
if (!checkConnect())
|
if (!checkConnect())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
@ -1028,7 +1041,6 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
||||||
_getFileRecordQuery->bindValue(1, getPHash(filename));
|
_getFileRecordQuery->bindValue(1, getPHash(filename));
|
||||||
|
|
||||||
if (!_getFileRecordQuery->exec()) {
|
if (!_getFileRecordQuery->exec()) {
|
||||||
locker.unlock();
|
|
||||||
close();
|
close();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -1040,9 +1052,7 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
|
||||||
if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE
|
if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE
|
||||||
QString err = _getFileRecordQuery->error();
|
QString err = _getFileRecordQuery->error();
|
||||||
qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err;
|
qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err;
|
||||||
locker.unlock();
|
|
||||||
close();
|
close();
|
||||||
locker.relock();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1058,7 +1068,7 @@ bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *r
|
||||||
rec->_path.clear();
|
rec->_path.clear();
|
||||||
Q_ASSERT(!rec->isValid());
|
Q_ASSERT(!rec->isValid());
|
||||||
|
|
||||||
if (!inode)
|
if (!inode || _metadataTableIsEmpty)
|
||||||
return true; // no error, yet nothing found (rec->isValid() == false)
|
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||||
|
|
||||||
if (!checkConnect())
|
if (!checkConnect())
|
||||||
|
@ -1087,7 +1097,7 @@ bool SyncJournalDb::getFileRecordByFileId(const QByteArray &fileId, SyncJournalF
|
||||||
rec->_path.clear();
|
rec->_path.clear();
|
||||||
Q_ASSERT(!rec->isValid());
|
Q_ASSERT(!rec->isValid());
|
||||||
|
|
||||||
if (fileId.isEmpty())
|
if (fileId.isEmpty() || _metadataTableIsEmpty)
|
||||||
return true; // no error, yet nothing found (rec->isValid() == false)
|
return true; // no error, yet nothing found (rec->isValid() == false)
|
||||||
|
|
||||||
if (!checkConnect())
|
if (!checkConnect())
|
||||||
|
@ -1111,6 +1121,9 @@ bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::functio
|
||||||
{
|
{
|
||||||
QMutexLocker locker(&_mutex);
|
QMutexLocker locker(&_mutex);
|
||||||
|
|
||||||
|
if (_metadataTableIsEmpty)
|
||||||
|
return true; // no error, yet nothing found
|
||||||
|
|
||||||
if (!checkConnect())
|
if (!checkConnect())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
@ -1184,15 +1197,11 @@ int SyncJournalDb::getFileRecordCount()
|
||||||
{
|
{
|
||||||
QMutexLocker locker(&_mutex);
|
QMutexLocker locker(&_mutex);
|
||||||
|
|
||||||
if (!checkConnect()) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
SqlQuery query(_db);
|
SqlQuery query(_db);
|
||||||
query.prepare("SELECT COUNT(*) FROM metadata");
|
query.prepare("SELECT COUNT(*) FROM metadata");
|
||||||
|
|
||||||
if (!query.exec()) {
|
if (!query.exec()) {
|
||||||
return 0;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (query.next()) {
|
if (query.next()) {
|
||||||
|
@ -1200,7 +1209,7 @@ int SyncJournalDb::getFileRecordCount()
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SyncJournalDb::updateFileRecordChecksum(const QString &filename,
|
bool SyncJournalDb::updateFileRecordChecksum(const QString &filename,
|
||||||
|
@ -1775,7 +1784,6 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path)
|
||||||
|
|
||||||
// We also need to remove the ETags so the update phase refreshes the directory paths
|
// We also need to remove the ETags so the update phase refreshes the directory paths
|
||||||
// on the next sync
|
// on the next sync
|
||||||
locker.unlock();
|
|
||||||
avoidReadFromDbOnNextSync(path);
|
avoidReadFromDbOnNextSync(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -66,7 +66,6 @@ public:
|
||||||
bool setFileRecordMetadata(const SyncJournalFileRecord &record);
|
bool setFileRecordMetadata(const SyncJournalFileRecord &record);
|
||||||
|
|
||||||
bool deleteFileRecord(const QString &filename, bool recursively = false);
|
bool deleteFileRecord(const QString &filename, bool recursively = false);
|
||||||
int getFileRecordCount();
|
|
||||||
bool updateFileRecordChecksum(const QString &filename,
|
bool updateFileRecordChecksum(const QString &filename,
|
||||||
const QByteArray &contentChecksum,
|
const QByteArray &contentChecksum,
|
||||||
const QByteArray &contentChecksumType);
|
const QByteArray &contentChecksumType);
|
||||||
|
@ -215,6 +214,7 @@ public:
|
||||||
void clearFileTable();
|
void clearFileTable();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
int getFileRecordCount();
|
||||||
bool updateDatabaseStructure();
|
bool updateDatabaseStructure();
|
||||||
bool updateMetadataTableStructure();
|
bool updateMetadataTableStructure();
|
||||||
bool updateErrorBlacklistTableStructure();
|
bool updateErrorBlacklistTableStructure();
|
||||||
|
@ -237,6 +237,7 @@ private:
|
||||||
QString _dbFile;
|
QString _dbFile;
|
||||||
QMutex _mutex; // Public functions are protected with the mutex.
|
QMutex _mutex; // Public functions are protected with the mutex.
|
||||||
int _transaction;
|
int _transaction;
|
||||||
|
bool _metadataTableIsEmpty;
|
||||||
|
|
||||||
// NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close
|
// NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close
|
||||||
QScopedPointer<SqlQuery> _getFileRecordQuery;
|
QScopedPointer<SqlQuery> _getFileRecordQuery;
|
||||||
|
|
|
@ -310,7 +310,6 @@ int csync_s::reinitialize() {
|
||||||
|
|
||||||
remote.read_from_db = 0;
|
remote.read_from_db = 0;
|
||||||
read_remote_from_db = true;
|
read_remote_from_db = true;
|
||||||
db_is_empty = false;
|
|
||||||
|
|
||||||
local.files.clear();
|
local.files.clear();
|
||||||
remote.files.clear();
|
remote.files.clear();
|
||||||
|
|
|
@ -142,12 +142,6 @@ struct OCSYNC_EXPORT csync_s {
|
||||||
*/
|
*/
|
||||||
bool read_remote_from_db = false;
|
bool read_remote_from_db = false;
|
||||||
|
|
||||||
/**
|
|
||||||
* If true, the DB is considered empty and all reads are skipped. (default is false)
|
|
||||||
* This is useful during the initial local discovery as it speeds it up significantly.
|
|
||||||
*/
|
|
||||||
bool db_is_empty = false;
|
|
||||||
|
|
||||||
bool ignore_hidden_files = true;
|
bool ignore_hidden_files = true;
|
||||||
|
|
||||||
csync_s(const char *localUri, OCC::SyncJournalDb *statedb);
|
csync_s(const char *localUri, OCC::SyncJournalDb *statedb);
|
||||||
|
|
|
@ -786,7 +786,6 @@ void SyncEngine::startSync()
|
||||||
|
|
||||||
csync_resume(_csync_ctx.data());
|
csync_resume(_csync_ctx.data());
|
||||||
|
|
||||||
int fileRecordCount = -1;
|
|
||||||
if (!_journal->exists()) {
|
if (!_journal->exists()) {
|
||||||
qCInfo(lcEngine) << "New sync (no sync journal exists)";
|
qCInfo(lcEngine) << "New sync (no sync journal exists)";
|
||||||
} else {
|
} else {
|
||||||
|
@ -800,9 +799,8 @@ void SyncEngine::startSync()
|
||||||
verStr.append(" on ").append(Utility::platformName());
|
verStr.append(" on ").append(Utility::platformName());
|
||||||
qCInfo(lcEngine) << verStr;
|
qCInfo(lcEngine) << verStr;
|
||||||
|
|
||||||
fileRecordCount = _journal->getFileRecordCount(); // this creates the DB if it does not exist yet
|
// This creates the DB if it does not exist yet.
|
||||||
|
if (!_journal->isConnected()) {
|
||||||
if (fileRecordCount == -1) {
|
|
||||||
qCWarning(lcEngine) << "No way to create a sync journal!";
|
qCWarning(lcEngine) << "No way to create a sync journal!";
|
||||||
csyncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder."));
|
csyncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder."));
|
||||||
finalize(false);
|
finalize(false);
|
||||||
|
@ -812,10 +810,6 @@ void SyncEngine::startSync()
|
||||||
|
|
||||||
_csync_ctx->read_remote_from_db = true;
|
_csync_ctx->read_remote_from_db = true;
|
||||||
|
|
||||||
// This tells csync to never read from the DB if it is empty
|
|
||||||
// thereby speeding up the initial discovery significantly.
|
|
||||||
_csync_ctx->db_is_empty = (fileRecordCount == 0);
|
|
||||||
|
|
||||||
bool ok;
|
bool ok;
|
||||||
auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok);
|
auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok);
|
||||||
if (ok) {
|
if (ok) {
|
||||||
|
|
Loading…
Reference in a new issue