diff --git a/src/common/checksums.cpp b/src/common/checksums.cpp index c604842a3..79038554f 100644 --- a/src/common/checksums.cpp +++ b/src/common/checksums.cpp @@ -328,7 +328,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) { qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader; - emit validationFailed(tr("The checksum header is malformed.")); + emit validationFailed(tr("The checksum header is malformed."), {}, {}, _filePath); return nullptr; } @@ -341,6 +341,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader) { + _filePath = filePath; if (auto calculator = prepareStart(checksumHeader)) calculator->start(filePath); } @@ -355,11 +356,11 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumTy const QByteArray &checksum) { if (checksumType != _expectedChecksumType) { - emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType))); + emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType)), {}, {}, _filePath); return; } if (checksum != _expectedChecksum) { - emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed. '%1' != '%2'").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum))); + emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed. '%1' != '%2'").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)), checksumType, checksum, _filePath); return; } emit validated(checksumType, checksum); diff --git a/src/common/checksums.h b/src/common/checksums.h index 5c8d39d5c..843017a43 100644 --- a/src/common/checksums.h +++ b/src/common/checksums.h @@ -163,7 +163,7 @@ public: signals: void validated(const QByteArray &checksumType, const QByteArray &checksum); - void validationFailed(const QString &errMsg); + void validationFailed(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath); private slots: void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum); @@ -173,6 +173,8 @@ private: QByteArray _expectedChecksumType; QByteArray _expectedChecksum; + + QString _filePath; }; /** diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 586573ba4..579289df5 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -976,7 +976,7 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) } } -void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value) +void SyncJournalDb::keyValueStoreSet(const QString &key, QVariant value) { QMutexLocker locker(&_mutex); if (!checkConnect()) { @@ -988,7 +988,7 @@ void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value) } _setKeyValueStoreQuery.bindValue(1, key); - _setKeyValueStoreQuery.bindValue(2, QString::number(value)); + _setKeyValueStoreQuery.bindValue(2, value); _setKeyValueStoreQuery.exec(); } @@ -1013,6 +1013,40 @@ qint64 SyncJournalDb::keyValueStoreGetInt(const QString &key, qint64 defaultValu return _getKeyValueStoreQuery.int64Value(0); } +QVariant SyncJournalDb::keyValueStoreGet(const QString &key, QVariant defaultValue) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return defaultValue; + } + + if (!_getKeyValueStoreQuery.initOrReset(QByteArrayLiteral("SELECT value FROM key_value_store WHERE key = ?1;"), _db)) { + return defaultValue; + } + + _getKeyValueStoreQuery.bindValue(1, key); + _getKeyValueStoreQuery.exec(); + + if (!_getKeyValueStoreQuery.next().hasData) { + return defaultValue; + } + + return _getKeyValueStoreQuery.stringValue(0); +} + +void SyncJournalDb::keyValueStoreDelete(const QString &key) +{ + if (!_deleteKeyValueStoreQuery.initOrReset("DELETE FROM key_value_store WHERE key=?1;", _db)) { + qCWarning(lcDb) << "Failed to initOrReset _deleteKeyValueStoreQuery"; + Q_ASSERT(false); + } + _deleteKeyValueStoreQuery.bindValue(1, key); + if (!_deleteKeyValueStoreQuery.exec()) { + qCWarning(lcDb) << "Failed to exec _deleteKeyValueStoreQuery for key" << key; + Q_ASSERT(false); + } +} + // TODO: filename -> QBytearray? bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively) { diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 29eb57def..cbfa8d387 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "common/utility.h" @@ -66,8 +67,10 @@ public: bool listFilesInPath(const QByteArray &path, const std::function &rowCallback); bool setFileRecord(const SyncJournalFileRecord &record); - void keyValueStoreSet(const QString &key, qint64 value); + void keyValueStoreSet(const QString &key, QVariant value); qint64 keyValueStoreGetInt(const QString &key, qint64 defaultValue); + QVariant keyValueStoreGet(const QString &key, QVariant defaultValue = {}); + void keyValueStoreDelete(const QString &key); bool deleteFileRecord(const QString &filename, bool recursively = false); bool updateFileRecordChecksum(const QString &filename, @@ -423,6 +426,7 @@ private: SqlQuery _setDataFingerprintQuery2; SqlQuery _setKeyValueStoreQuery; SqlQuery _getKeyValueStoreQuery; + SqlQuery _deleteKeyValueStoreQuery; SqlQuery _getConflictRecordQuery; SqlQuery _setConflictRecordQuery; SqlQuery _deleteConflictRecordQuery; diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index a8b5c76ca..954018a19 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -49,6 +49,10 @@ #define DEFAULT_REMOTE_POLL_INTERVAL 30000 // default remote poll time in milliseconds #define DEFAULT_MAX_LOG_LINES 20000 +namespace { + static constexpr char allowChecksumValidationFailC[] = "allowChecksumValidationFail"; +} + namespace OCC { namespace chrono = std::chrono; @@ -101,7 +105,6 @@ static const char useNewBigFolderSizeLimitC[] = "useNewBigFolderSizeLimit"; static const char confirmExternalStorageC[] = "confirmExternalStorage"; static const char moveToTrashC[] = "moveToTrash"; - const char certPath[] = "http_certificatePath"; const char certPasswd[] = "http_certificatePasswd"; QString ConfigFile::_confDir = QString(); @@ -113,7 +116,6 @@ static chrono::milliseconds millisecondsValue(const QSettings &setting, const ch return chrono::milliseconds(setting.value(QLatin1String(key), qlonglong(defaultValue.count())).toLongLong()); } - bool copy_dir_recursive(QString from_dir, QString to_dir) { QDir dir; @@ -889,6 +891,11 @@ void ConfigFile::setMoveToTrash(bool isChecked) setValue(moveToTrashC, isChecked); } +bool ConfigFile::allowChecksumValidationFail() const +{ + return getValue(allowChecksumValidationFailC, {}, false).toBool(); +} + bool ConfigFile::promptDeleteFiles() const { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index cf33d42b4..537bfec0c 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -145,6 +145,9 @@ public: bool moveToTrash() const; void setMoveToTrash(bool); + /** should we allow checksum validation to fail? set to true to workaround corrupted checksums **/ + bool allowChecksumValidationFail() const; + static bool setConfDir(const QString &value); bool optionalServerNotifications() const; diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index c5040cb59..d37fbeaf3 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -28,6 +28,8 @@ #include "propagatedownloadencrypted.h" #include "common/vfs.h" +#include "configfile.h" + #include #include #include @@ -38,6 +40,11 @@ #include #endif +namespace { + constexpr quint16 numChecksumFailuresAllowed = 1; + constexpr char *checksumFailureDbRecordPrefix = "ChecksumValidationFailed_"; +} + namespace OCC { Q_LOGGING_CATEGORY(lcGetJob, "nextcloud.sync.networkjob.get", QtInfoMsg) @@ -821,8 +828,38 @@ void PropagateDownloadFile::slotGetFinished() validator->start(_tmpFile.fileName(), checksumHeader); } -void PropagateDownloadFile::slotChecksumFail(const QString &errMsg) +void PropagateDownloadFile::slotChecksumFail(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath) { + if (!checksumType.isEmpty() && !checksum.isEmpty() && !filePath.isEmpty()) { + ConfigFile cfgFile; + + if (cfgFile.allowChecksumValidationFail()) { + const auto key = QString(checksumFailureDbRecordPrefix + _item->_fileId); + const QStringList mismatchEntryForFileSplitted = propagator()->_journal->keyValueStoreGet(key).toString().split(":", QString::SkipEmptyParts); + const QByteArray mismatchChecksumForFile = mismatchEntryForFileSplitted.size() > 0 ? mismatchEntryForFileSplitted[0].toUtf8() : QByteArray(); + const auto numChecksumMismatchCases = mismatchEntryForFileSplitted.size() > 1 ? mismatchEntryForFileSplitted[1].toInt() : 0; + + // format must be CHECKSUM:COUNT + Q_ASSERT(mismatchEntryForFileSplitted.size() != 1); + if (mismatchEntryForFileSplitted.size() == 1) { + qCCritical(lcPropagateDownload) << "mismatchEntryForFile has incorrect format. Should be CHECKSUM:COUNT"; + } + + if (numChecksumMismatchCases < numChecksumFailuresAllowed || mismatchChecksumForFile != checksum) { + // not enough failures or different checksum this time + qCInfo(lcPropagateDownload) << "Checksum validation has failed" << numChecksumMismatchCases << " times, with previous checksum<" << mismatchChecksumForFile << "> and, current checksum<" << checksum << ">, but, allowChecksumValidationFail is set.Let's give it another try..."; + const auto numCasesToSet = mismatchChecksumForFile != checksum ? 1 : numChecksumMismatchCases + 1; + const QString value = QString::fromUtf8(checksum) + QStringLiteral(":") + QString::number(numCasesToSet); + propagator()->_journal->keyValueStoreSet(key, value); + } else { + propagator()->_journal->keyValueStoreDelete(key); + qCInfo(lcPropagateDownload) << "Checksum validation has failed" << numChecksumMismatchCases << " times, but, allowChecksumValidationFail is set, so, let's continue..."; + startContentChecksumCompute(checksumType, filePath); + return; + } + } + } + FileSystem::remove(_tmpFile.fileName()); propagator()->_anotherSyncNeeded = true; done(SyncFileItem::SoftError, errMsg); // tr("The file downloaded with a broken checksum, will be redownloaded.")); @@ -850,6 +887,18 @@ void PropagateDownloadFile::deleteExistingFolder() } } +void PropagateDownloadFile::startContentChecksumCompute(const QByteArray &checksumType, const QString &path) +{ + qCInfo(lcPropagateDownload) << "Start checksum compute with checksumType:" << checksumType << " for path:" << path; + // Compute the content checksum. + const auto computeChecksum = new ComputeChecksum(this); + computeChecksum->setChecksumType(checksumType); + + connect(computeChecksum, &ComputeChecksum::done, + this, &PropagateDownloadFile::contentChecksumComputed); + computeChecksum->start(path); +} + namespace { // Anonymous namespace for the recall feature static QString makeRecallFileName(const QString &fn) { @@ -938,13 +987,9 @@ void PropagateDownloadFile::transmissionChecksumValidated(const QByteArray &chec return contentChecksumComputed(checksumType, checksum); } - // Compute the content checksum. - auto computeChecksum = new ComputeChecksum(this); - computeChecksum->setChecksumType(theContentChecksumType); + startContentChecksumCompute(theContentChecksumType, _tmpFile.fileName()); - connect(computeChecksum, &ComputeChecksum::done, - this, &PropagateDownloadFile::contentChecksumComputed); - computeChecksum->start(_tmpFile.fileName()); + propagator()->_journal->keyValueStoreDelete(QString(checksumFailureDbRecordPrefix + _item->_fileId)); } void PropagateDownloadFile::contentChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum) diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 06c9cc7f1..769d28eb1 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -202,12 +202,14 @@ private slots: void abort(PropagatorJob::AbortType abortType) override; void slotDownloadProgress(qint64, qint64); - void slotChecksumFail(const QString &errMsg); + void slotChecksumFail(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath); private: void startAfterIsEncryptedIsChecked(); void deleteExistingFolder(); + void startContentChecksumCompute(const QByteArray &checksumType, const QString &path); + qint64 _resumeStart; qint64 _downloadProgress; QPointer _job; diff --git a/test/testchecksumvalidator.cpp b/test/testchecksumvalidator.cpp index 93af94bde..a7e07d85b 100644 --- a/test/testchecksumvalidator.cpp +++ b/test/testchecksumvalidator.cpp @@ -42,7 +42,7 @@ using namespace OCC::Utility; _successDown = true; } - void slotDownError( const QString& errMsg ) { + void slotDownError(const QString &errMsg, const QByteArray&, const QByteArray&, const QString&) { QCOMPARE(_expectedError, errMsg); _errorSeen = true; } @@ -179,8 +179,8 @@ using namespace OCC::Utility; QSKIP("ZLIB not found.", SkipSingle); #else auto *vali = new ValidateChecksumHeader(this); - connect(vali, SIGNAL(validated(QByteArray,QByteArray)), this, SLOT(slotDownValidated())); - connect(vali, SIGNAL(validationFailed(QString)), this, SLOT(slotDownError(QString))); + connect(vali, &ValidateChecksumHeader::validated, this, &TestChecksumValidator::slotDownValidated); + connect(vali, &ValidateChecksumHeader::validationFailed, this, &TestChecksumValidator::slotDownError); auto file = new QFile(_testfile, vali); file->open(QIODevice::ReadOnly);