2015-05-15 16:34:17 +03:00
|
|
|
/*
|
|
|
|
* Copyright (C) by Klaas Freitag <freitag@owncloud.com>
|
|
|
|
*
|
2017-09-01 19:11:43 +03:00
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
2015-05-15 16:34:17 +03:00
|
|
|
*
|
2017-09-01 19:11:43 +03:00
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
2015-05-15 16:34:17 +03:00
|
|
|
*/
|
|
|
|
#include "config.h"
|
2017-09-01 19:11:43 +03:00
|
|
|
#include "filesystembase.h"
|
|
|
|
#include "common/checksums.h"
|
2015-05-15 16:34:17 +03:00
|
|
|
|
2017-05-09 15:24:11 +03:00
|
|
|
#include <QLoggingCategory>
|
2015-05-22 15:43:47 +03:00
|
|
|
#include <qtconcurrentrun.h>
|
2015-05-15 16:34:17 +03:00
|
|
|
|
2015-12-08 12:59:42 +03:00
|
|
|
/** \file checksums.cpp
|
|
|
|
*
|
|
|
|
* \brief Computing and validating file checksums
|
|
|
|
*
|
|
|
|
* Overview
|
|
|
|
* --------
|
|
|
|
*
|
|
|
|
* Checksums are used in two distinct ways during synchronization:
|
|
|
|
*
|
|
|
|
* - to guard uploads and downloads against data corruption
|
|
|
|
* (transmission checksum)
|
|
|
|
* - to quickly check whether the content of a file has changed
|
|
|
|
* to avoid redundant uploads (content checksum)
|
|
|
|
*
|
|
|
|
* In principle both are independent and different checksumming
|
|
|
|
* algorithms can be used. To avoid redundant computations, it can
|
|
|
|
* make sense to use the same checksum algorithm though.
|
|
|
|
*
|
|
|
|
* Transmission Checksums
|
|
|
|
* ----------------------
|
|
|
|
*
|
|
|
|
* The usage of transmission checksums is currently optional and needs
|
|
|
|
* to be explicitly enabled by adding 'transmissionChecksum=TYPE' to
|
|
|
|
* the '[General]' section of the config file.
|
|
|
|
*
|
|
|
|
* When enabled, the checksum will be calculated on upload and sent to
|
|
|
|
* the server in the OC-Checksum header with the format 'TYPE:CHECKSUM'.
|
|
|
|
*
|
|
|
|
* On download, the header with the same name is read and if the
|
|
|
|
* received data does not have the expected checksum, the download is
|
|
|
|
* rejected.
|
|
|
|
*
|
|
|
|
* Transmission checksums guard a specific sync action and are not stored
|
|
|
|
* in the database.
|
|
|
|
*
|
|
|
|
* Content Checksums
|
|
|
|
* -----------------
|
|
|
|
*
|
|
|
|
* Sometimes the metadata of a local file changes while the content stays
|
|
|
|
* unchanged. Content checksums allow the sync client to avoid uploading
|
|
|
|
* the same data again by comparing the file's actual checksum to the
|
|
|
|
* checksum stored in the database.
|
|
|
|
*
|
|
|
|
* Content checksums are not sent to the server.
|
|
|
|
*
|
|
|
|
* Checksum Algorithms
|
|
|
|
* -------------------
|
|
|
|
*
|
|
|
|
* - Adler32 (requires zlib)
|
|
|
|
* - MD5
|
|
|
|
* - SHA1
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2015-05-15 16:34:17 +03:00
|
|
|
namespace OCC {
|
|
|
|
|
2017-12-28 22:33:10 +03:00
|
|
|
Q_LOGGING_CATEGORY(lcChecksums, "nextcloud.sync.checksums", QtInfoMsg)
|
2017-05-09 15:24:11 +03:00
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum)
|
|
|
|
{
|
2017-06-14 13:14:46 +03:00
|
|
|
if (checksumType.isEmpty() || checksum.isEmpty())
|
|
|
|
return QByteArray();
|
2015-10-14 16:03:40 +03:00
|
|
|
QByteArray header = checksumType;
|
|
|
|
header.append(':');
|
|
|
|
header.append(checksum);
|
|
|
|
return header;
|
|
|
|
}
|
|
|
|
|
2017-11-20 10:18:52 +03:00
|
|
|
QByteArray findBestChecksum(const QByteArray &checksums)
|
|
|
|
{
|
|
|
|
int i = 0;
|
|
|
|
// The order of the searches here defines the preference ordering.
|
|
|
|
if (-1 != (i = checksums.indexOf("SHA1:"))
|
|
|
|
|| -1 != (i = checksums.indexOf("MD5:"))
|
|
|
|
|| -1 != (i = checksums.indexOf("Adler32:"))) {
|
|
|
|
// Now i is the start of the best checksum
|
|
|
|
// Grab it until the next space or end of string.
|
|
|
|
auto checksum = checksums.mid(i);
|
|
|
|
return checksum.mid(0, checksum.indexOf(" "));
|
|
|
|
}
|
|
|
|
return QByteArray();
|
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum)
|
|
|
|
{
|
|
|
|
if (header.isEmpty()) {
|
|
|
|
type->clear();
|
|
|
|
checksum->clear();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
const auto idx = header.indexOf(':');
|
|
|
|
if (idx < 0) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
*type = header.left(idx);
|
|
|
|
*checksum = header.mid(idx + 1);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-06-14 13:14:46 +03:00
|
|
|
|
|
|
|
QByteArray parseChecksumHeaderType(const QByteArray &header)
|
|
|
|
{
|
|
|
|
const auto idx = header.indexOf(':');
|
|
|
|
if (idx < 0) {
|
|
|
|
return QByteArray();
|
|
|
|
}
|
|
|
|
return header.left(idx);
|
|
|
|
}
|
|
|
|
|
2015-10-15 10:39:49 +03:00
|
|
|
bool uploadChecksumEnabled()
|
|
|
|
{
|
2017-09-29 11:31:41 +03:00
|
|
|
static bool enabled = qEnvironmentVariableIsEmpty("OWNCLOUD_DISABLE_CHECKSUM_UPLOAD");
|
2015-10-15 10:39:49 +03:00
|
|
|
return enabled;
|
|
|
|
}
|
|
|
|
|
2016-03-02 15:52:14 +03:00
|
|
|
QByteArray contentChecksumType()
|
|
|
|
{
|
|
|
|
static QByteArray type = qgetenv("OWNCLOUD_CONTENT_CHECKSUM_TYPE");
|
2017-01-26 14:17:00 +03:00
|
|
|
if (type.isNull()) { // can set to "" to disable checksumming
|
|
|
|
type = "SHA1";
|
2016-03-02 15:52:14 +03:00
|
|
|
}
|
2017-01-26 14:17:00 +03:00
|
|
|
return type;
|
2016-03-02 15:52:14 +03:00
|
|
|
}
|
|
|
|
|
2017-10-13 14:08:20 +03:00
|
|
|
static bool checksumComputationEnabled()
|
|
|
|
{
|
2020-03-21 03:05:45 +03:00
|
|
|
static bool enabled = qEnvironmentVariableIsEmpty("OWNCLOUD_DISABLE_CHECKSUM_COMPUTATIONS");
|
2017-10-13 14:08:20 +03:00
|
|
|
return enabled;
|
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
ComputeChecksum::ComputeChecksum(QObject *parent)
|
|
|
|
: QObject(parent)
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
void ComputeChecksum::setChecksumType(const QByteArray &type)
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
|
|
|
_checksumType = type;
|
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
QByteArray ComputeChecksum::checksumType() const
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
2015-10-01 16:00:33 +03:00
|
|
|
return _checksumType;
|
2015-05-21 15:30:21 +03:00
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
void ComputeChecksum::start(const QString &filePath)
|
2015-05-21 15:30:21 +03:00
|
|
|
{
|
2017-10-13 14:08:20 +03:00
|
|
|
qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread";
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
// Calculate the checksum in a different thread first.
|
2017-09-20 11:14:48 +03:00
|
|
|
connect(&_watcher, &QFutureWatcherBase::finished,
|
|
|
|
this, &ComputeChecksum::slotCalculationDone,
|
2015-10-14 16:03:40 +03:00
|
|
|
Qt::UniqueConnection);
|
2015-11-23 13:53:06 +03:00
|
|
|
_watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType()));
|
|
|
|
}
|
2015-10-14 16:03:40 +03:00
|
|
|
|
2015-11-23 13:53:06 +03:00
|
|
|
QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray &checksumType)
|
|
|
|
{
|
2017-10-13 14:08:20 +03:00
|
|
|
if (!checksumComputationEnabled()) {
|
|
|
|
qCWarning(lcChecksums) << "Checksum computation disabled by environment variable";
|
|
|
|
return QByteArray();
|
|
|
|
}
|
|
|
|
|
2015-11-23 13:53:06 +03:00
|
|
|
if (checksumType == checkSumMD5C) {
|
|
|
|
return FileSystem::calcMd5(filePath);
|
|
|
|
} else if (checksumType == checkSumSHA1C) {
|
|
|
|
return FileSystem::calcSha1(filePath);
|
2015-10-14 16:03:40 +03:00
|
|
|
}
|
2015-05-15 16:34:17 +03:00
|
|
|
#ifdef ZLIB_FOUND
|
2015-11-23 13:53:06 +03:00
|
|
|
else if (checksumType == checkSumAdlerC) {
|
|
|
|
return FileSystem::calcAdler32(filePath);
|
2015-10-14 16:03:40 +03:00
|
|
|
}
|
2015-05-15 16:34:17 +03:00
|
|
|
#endif
|
2015-11-23 13:53:06 +03:00
|
|
|
// for an unknown checksum or no checksum, we're done right now
|
|
|
|
if (!checksumType.isEmpty()) {
|
2017-03-30 14:46:20 +03:00
|
|
|
qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType;
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
2015-11-23 13:53:06 +03:00
|
|
|
return QByteArray();
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
void ComputeChecksum::slotCalculationDone()
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
|
|
|
QByteArray checksum = _watcher.future().result();
|
2015-11-23 13:53:06 +03:00
|
|
|
if (!checksum.isNull()) {
|
|
|
|
emit done(_checksumType, checksum);
|
|
|
|
} else {
|
|
|
|
emit done(QByteArray(), QByteArray());
|
|
|
|
}
|
2015-10-14 16:03:40 +03:00
|
|
|
}
|
2015-05-15 16:34:17 +03:00
|
|
|
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent)
|
|
|
|
: QObject(parent)
|
|
|
|
{
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader)
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
2015-10-14 16:03:40 +03:00
|
|
|
// If the incoming header is empty no validation can happen. Just continue.
|
2015-10-14 14:41:51 +03:00
|
|
|
if (checksumHeader.isEmpty()) {
|
2015-10-28 13:00:03 +03:00
|
|
|
emit validated(QByteArray(), QByteArray());
|
2015-05-15 16:34:17 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
|
2017-03-30 14:46:20 +03:00
|
|
|
qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
|
2015-05-19 17:49:22 +03:00
|
|
|
emit validationFailed(tr("The checksum header is malformed."));
|
|
|
|
return;
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
2015-10-14 16:03:40 +03:00
|
|
|
|
|
|
|
auto calculator = new ComputeChecksum(this);
|
|
|
|
calculator->setChecksumType(_expectedChecksumType);
|
2017-09-20 11:14:48 +03:00
|
|
|
connect(calculator, &ComputeChecksum::done,
|
|
|
|
this, &ValidateChecksumHeader::slotChecksumCalculated);
|
2015-10-14 16:03:40 +03:00
|
|
|
calculator->start(filePath);
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
|
|
|
|
2015-10-14 16:03:40 +03:00
|
|
|
void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType,
|
|
|
|
const QByteArray &checksum)
|
2015-05-15 16:34:17 +03:00
|
|
|
{
|
2015-10-14 16:03:40 +03:00
|
|
|
if (checksumType != _expectedChecksumType) {
|
|
|
|
emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType)));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (checksum != _expectedChecksum) {
|
2015-05-19 17:49:22 +03:00
|
|
|
emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed."));
|
2015-10-14 16:03:40 +03:00
|
|
|
return;
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
2015-10-28 13:00:03 +03:00
|
|
|
emit validated(checksumType, checksum);
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|
|
|
|
|
2020-05-25 22:33:24 +03:00
|
|
|
CSyncChecksumHook::CSyncChecksumHook() = default;
|
2015-11-23 13:53:06 +03:00
|
|
|
|
2017-08-17 11:06:14 +03:00
|
|
|
QByteArray CSyncChecksumHook::hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void * /*this_obj*/)
|
2015-11-23 13:53:06 +03:00
|
|
|
{
|
2017-06-14 13:14:46 +03:00
|
|
|
QByteArray type = parseChecksumHeaderType(QByteArray(otherChecksumHeader));
|
|
|
|
if (type.isEmpty())
|
2020-05-18 21:39:16 +03:00
|
|
|
return nullptr;
|
2017-06-14 13:14:46 +03:00
|
|
|
|
2017-10-13 14:08:20 +03:00
|
|
|
qCInfo(lcChecksums) << "Computing" << type << "checksum of" << path << "in the csync hook";
|
2017-08-17 11:06:14 +03:00
|
|
|
QByteArray checksum = ComputeChecksum::computeNow(QString::fromUtf8(path), type);
|
2015-11-23 15:44:49 +03:00
|
|
|
if (checksum.isNull()) {
|
2017-06-14 13:14:46 +03:00
|
|
|
qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path;
|
2020-05-18 21:39:16 +03:00
|
|
|
return nullptr;
|
2015-11-23 15:44:49 +03:00
|
|
|
}
|
|
|
|
|
2017-08-17 11:06:14 +03:00
|
|
|
return makeChecksumHeader(type, checksum);
|
2015-11-23 13:53:06 +03:00
|
|
|
}
|
|
|
|
|
2015-05-15 16:34:17 +03:00
|
|
|
}
|