From 64ce0cd7a212fcda54a80e32c3d76cfd3b8db548 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 20 May 2015 21:42:08 +0200 Subject: [PATCH] Improved implementation efficiency of checksum calculation. Based on Oliviers suggestions the file read routine now reuses one buffer to read the entire file. Other cleanups. --- src/libsync/filesystem.cpp | 53 +++++++++++++++---------------- src/libsync/filesystem.h | 9 +++--- test/testtranschecksumvalidator.h | 15 ++++----- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/src/libsync/filesystem.cpp b/src/libsync/filesystem.cpp index c1c20d79a..b3fb52657 100644 --- a/src/libsync/filesystem.cpp +++ b/src/libsync/filesystem.cpp @@ -11,7 +11,6 @@ * for more details. */ -#include "config.h" #include "filesystem.h" #include "utility.h" @@ -20,8 +19,6 @@ #include #include #include -#include -#include #ifdef ZLIB_FOUND #include @@ -402,57 +399,57 @@ QString FileSystem::fileSystemForPath(const QString & path) } #endif -QByteArray FileSystem::calcMd5( const QString& filename ) -{ - QByteArray arr; +#define BUFSIZE 1024*1024*10 - QCryptographicHash crypto( QCryptographicHash::Md5 ); +static QByteArray readToCrypto( const QString& filename, QCryptographicHash::Algorithm algo ) +{ + const qint64 bufSize = BUFSIZE; + QByteArray buf(bufSize,0); + QByteArray arr; + QCryptographicHash crypto( algo ); QFile file(filename); if (file.open(QIODevice::ReadOnly)) { - QByteArray data; + qint64 size; while (!file.atEnd()) { - data = file.read(1024*1024*10); - crypto.addData(data); + size = file.read( buf.data(), bufSize ); + if( size > 0 ) { + crypto.addData(buf.data(), size); + } } arr = crypto.result().toHex(); } return arr; } +QByteArray FileSystem::calcMd5( const QString& filename ) +{ + return readToCrypto( filename, QCryptographicHash::Md5 ); +} + QByteArray FileSystem::calcSha1( const QString& filename ) { - QByteArray arr; - - QCryptographicHash crypto( QCryptographicHash::Sha1 ); - - QFile file(filename); - if (file.open(QIODevice::ReadOnly)) { - QByteArray data; - while (!file.atEnd()) { - data = file.read(1024*1024*10); - crypto.addData(data); - } - arr = crypto.result().toHex(); - } - return arr; + return readToCrypto( filename, QCryptographicHash::Sha1 ); } #ifdef ZLIB_FOUND QByteArray FileSystem::calcAdler32( const QString& filename ) { unsigned int adler = adler32(0L, Z_NULL, 0); + const qint64 bufSize = BUFSIZE; + QByteArray buf(bufSize, 0); QFile file(filename); if (file.open(QIODevice::ReadOnly)) { - QByteArray data; + qint64 size; while (!file.atEnd()) { - data = file.read(1024*1024*10); - adler = adler32(adler, (const Bytef*) data.data(), data.size()); + size = file.read(buf.data(), bufSize); + if( size > 0 ) + adler = adler32(adler, (const Bytef*) buf.data(), size); } } - return QString::number( adler, 16 ).toUtf8(); + return QByteArray::number( adler, 16 ); } #endif diff --git a/src/libsync/filesystem.h b/src/libsync/filesystem.h index 6e9b45c56..9e314a191 100644 --- a/src/libsync/filesystem.h +++ b/src/libsync/filesystem.h @@ -13,8 +13,11 @@ #pragma once +#include "config.h" + #include #include +#include #include @@ -123,12 +126,8 @@ QString fileSystemForPath(const QString & path); QByteArray calcMd5( const QString& fileName ); QByteArray calcSha1( const QString& fileName ); -QByteArray calcAdler32( const QString& fileName ); - #ifdef ZLIB_FOUND -QByteArray calcAdler32Worker( const QString& filename ); +QByteArray calcAdler32( const QString& fileName ); #endif -QByteArray calcSha1Worker( const QString& filename ); -QByteArray calcMd5Worker( const QString& filename ); }} diff --git a/test/testtranschecksumvalidator.h b/test/testtranschecksumvalidator.h index c43dd90bc..58aa52df3 100644 --- a/test/testtranschecksumvalidator.h +++ b/test/testtranschecksumvalidator.h @@ -72,24 +72,24 @@ using namespace OCC; void testUploadChecksummingAdler() { - TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile); + TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this); vali->setChecksumType("Adler32"); connect(vali, SIGNAL(validated()), this, SLOT(slotUpValidated())); - _expected = "Adler32:"+FileSystem::calcAdler32( _testfile ); + QString testfile = _testfile; + _expected = "Adler32:"+FileSystem::calcAdler32( testfile ); qDebug() << "XX Expected Checksum: " << _expected; vali->uploadValidation(_item); usleep(5000); _loop.processEvents(); - vali->deleteLater(); } void testUploadChecksummingMd5() { - TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile); + TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this); vali->setChecksumType( OCC::checkSumMD5C ); connect(vali, SIGNAL(validated()), this, SLOT(slotUpValidated())); @@ -100,12 +100,11 @@ using namespace OCC; usleep(2000); _loop.processEvents(); - vali->deleteLater(); } void testUploadChecksummingSha1() { - TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile); + TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this); vali->setChecksumType( OCC::checkSumSHA1C ); connect(vali, SIGNAL(validated()), this, SLOT(slotUpValidated())); @@ -117,7 +116,6 @@ using namespace OCC; usleep(2000); _loop.processEvents(); - vali->deleteLater(); } void testDownloadChecksummingAdler() { @@ -127,7 +125,7 @@ using namespace OCC; adler.append(FileSystem::calcAdler32( _testfile )); _successDown = false; - TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile); + TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this); connect(vali, SIGNAL(validated()), this, SLOT(slotDownValidated())); connect(vali, SIGNAL(validationFailed(QString)), this, SLOT(slotDownError(QString))); vali->downloadValidation(adler); @@ -151,7 +149,6 @@ using namespace OCC; _loop.processEvents(); QVERIFY(_errorSeen); - vali->deleteLater(); }