mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-10-27 23:17:13 +03:00
Code Issues Projects Releases Packages Wiki Activity

Upload: refactor the upload in two classes so the new chuning can be implemented

Browse source
This commit is contained in:
Olivier Goffart 2016-05-20 10:26:02 +02:00
parent 86eab48981
commit 4f3f642da6
5 changed files with 534 additions and 440 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/libsync/CMakeLists.txt
View file

@ -51,6 +51,7 @@ set(libsync_SRCS
propagatorjobs.cpp
propagatedownload.cpp
propagateupload.cpp
propagateuploadv1.cpp
propagateremotedelete.cpp
propagateremotemove.cpp
propagateremotemkdir.cpp

2
src/libsync/owncloudpropagator.cpp
View file

@ -270,7 +270,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItemPtr &item) {
job->setDeleteExistingFolder(deleteExisting);
return job;
} else {
auto job = new PropagateUploadFile(this, item);
auto job = new PropagateUploadFileV1(this, item); // FIXME, depending on server version, use the other chunking algorithm
job->setDeleteExisting(deleteExisting);
return job;
}

463
src/libsync/propagateupload.cpp
View file

@ -184,7 +184,13 @@ bool PollJob::finished()
return true;
}
void PropagateUploadFile::start()
void PropagateUploadFileCommon::setDeleteExisting(bool enabled)
{
_deleteExisting = enabled;
}
void PropagateUploadFileCommon::start()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
@ -205,7 +211,7 @@ void PropagateUploadFile::start()
job->start();
}
void PropagateUploadFile::slotComputeContentChecksum()
void PropagateUploadFileCommon::slotComputeContentChecksum()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
@ -239,12 +245,7 @@ void PropagateUploadFile::slotComputeContentChecksum()
computeChecksum->start(filePath);
}
void PropagateUploadFile::setDeleteExisting(bool enabled)
{
_deleteExisting = enabled;
}
void PropagateUploadFile::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum)
void PropagateUploadFileCommon::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum)
{
_item->_contentChecksum = contentChecksum;
_item->_contentChecksumType = contentChecksumType;
@ -276,7 +277,7 @@ void PropagateUploadFile::slotComputeTransmissionChecksum(const QByteArray& cont
computeChecksum->start(filePath);
}
void PropagateUploadFile::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
void PropagateUploadFileCommon::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
{
// Remove ourselfs from the list of active job, before any posible call to done()
// When we start chunks, we will add it again, once for every chunks.
@ -322,23 +323,7 @@ void PropagateUploadFile::slotStartUpload(const QByteArray& transmissionChecksum
return;
}
_chunkCount = std::ceil(fileSize/double(chunkSize()));
_startChunk = 0;
_transferId = qrand() ^ _item->_modtime ^ (_item->_size << 16);
const SyncJournalDb::UploadInfo progressInfo = _propagator->_journal->getUploadInfo(_item->_file);
if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item->_modtime ) {
_startChunk = progressInfo._chunk;
_transferId = progressInfo._transferid;
qDebug() << Q_FUNC_INFO << _item->_file << ": Resuming from chunk " << _startChunk;
}
_currentChunk = 0;
_duration.start();
emit progress(*_item, 0);
this->startNextChunk();
doStartUpload();
}
UploadDevice::UploadDevice(BandwidthManager *bwm)
@ -476,24 +461,64 @@ void UploadDevice::setChoked(bool b) {
}
}
void PropagateUploadFile::startNextChunk()
void PropagateUploadFileCommon::startPollJob(const QString& path)
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
PollJob* job = new PollJob(_propagator->account(), path, _item,
_propagator->_journal, _propagator->_localDir, this);
connect(job, SIGNAL(finishedSignal()), SLOT(slotPollFinished()));
SyncJournalDb::PollInfo info;
info._file = _item->_file;
info._url = path;
info._modtime = _item->_modtime;
_propagator->_journal->setPollInfo(info);
_propagator->_journal->commit("add poll info");
_propagator->_activeJobList.append(this);
job->start();
}
if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
// We return now and when the _jobs are finished we will proceed with the last chunk
// NOTE: Some other parts of the code such as slotUploadProgress also assume that the last chunk
// is sent last.
void PropagateUploadFileCommon::slotPollFinished()
{
PollJob *job = qobject_cast<PollJob *>(sender());
Q_ASSERT(job);
_propagator->_activeJobList.removeOne(this);
if (job->_item->_status != SyncFileItem::Success) {
_finished = true;
done(job->_item->_status, job->_item->_errorString);
return;
}
quint64 fileSize = _item->_size;
finalize();
}
void PropagateUploadFileCommon::slotJobDestroyed(QObject* job)
{
_jobs.erase(std::remove(_jobs.begin(), _jobs.end(), job) , _jobs.end());
}
void PropagateUploadFileCommon::abort()
{
foreach(auto *job, _jobs) {
if (job->reply()) {
qDebug() << Q_FUNC_INFO << job << this->_item->_file;
job->reply()->abort();
}
}
}
// This function is used whenever there is an error occuring and jobs might be in progress
void PropagateUploadFileCommon::abortWithError(SyncFileItem::Status status, const QString &error)
{
_finished = true;
abort();
done(status, error);
}
QMap<QByteArray, QByteArray> PropagateUploadFileCommon::headers()
{
QMap<QByteArray, QByteArray> headers;
headers["OC-Total-Length"] = QByteArray::number(fileSize);
headers["OC-Async"] = "1";
headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize()));
headers["Content-Type"] = "application/octet-stream";
headers["X-OC-Mtime"] = QByteArray::number(qint64(_item->_modtime));
@ -517,283 +542,12 @@ void PropagateUploadFile::startNextChunk()
// csync_owncloud.c's owncloud_file_id always strips the quotes.
headers["If-Match"] = '"' + _item->_etag + '"';
}
QString path = _item->_file;
UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager);
qint64 chunkStart = 0;
qint64 currentChunkSize = fileSize;
bool isFinalChunk = false;
if (_chunkCount > 1) {
int sendingChunk = (_currentChunk + _startChunk) % _chunkCount;
// XOR with chunk size to make sure everything goes well if chunk size changes between runs
uint transid = _transferId ^ chunkSize();
qDebug() << "Upload chunk" << sendingChunk << "of" << _chunkCount << "transferid(remote)=" << transid;
path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk);
headers["OC-Chunked"] = "1";
chunkStart = chunkSize() * quint64(sendingChunk);
currentChunkSize = chunkSize();
if (sendingChunk == _chunkCount - 1) { // last chunk
currentChunkSize = (fileSize % chunkSize());
if( currentChunkSize == 0 ) { // if the last chunk pretends to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
isFinalChunk = true;
}
} else {
// if there's only one chunk, it's the final one
isFinalChunk = true;
}
if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
headers[checkSumHeaderC] = makeChecksumHeader(
_transmissionChecksumType, _transmissionChecksum);
}
const QString fileName = _propagator->getFilePath(_item->_file);
if (! device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
qDebug() << "ERR: Could not prepare upload device: " << device->errorString();
// If the file is currently locked, we want to retry the sync
// when it becomes available again.
if (FileSystem::isFileLocked(fileName)) {
emit _propagator->seenLockedFile(fileName);
}
// Soft error because this is likely caused by the user modifying his files while syncing
abortWithError( SyncFileItem::SoftError, device->errorString() );
delete device;
return;
}
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk);
_jobs.append(job);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished()));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64)));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64)));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_propagator->_activeJobList.append(this);
_currentChunk++;
bool parallelChunkUpload = true;
QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK");
if (!env.isEmpty()) {
parallelChunkUpload = env != "false" && env != "0";
} else {
int versionNum = _propagator->account()->serverVersionInt();
if (versionNum < 0x080003) {
// Disable parallel chunk upload severs older than 8.0.3 to avoid too many
// internal sever errors (#2743, #2938)
parallelChunkUpload = false;
}
}
if (_currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
parallelChunkUpload = false;
}
if (parallelChunkUpload && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob())
&& _currentChunk < _chunkCount ) {
startNextChunk();
}
if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) {
emit ready();
}
return headers;
}
void PropagateUploadFile::slotPutFinished()
void PropagateUploadFileCommon::finalize()
{
PUTFileJob *job = qobject_cast<PUTFileJob *>(sender());
Q_ASSERT(job);
slotJobDestroyed(job); // remove it from the _jobs list
qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS"
<< job->reply()->error()
<< (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString())
<< job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute)
<< job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute);
_propagator->_activeJobList.removeOne(this);
if (_finished) {
// We have sent the finished signal already. We don't need to handle any remaining jobs
return;
}
QNetworkReply::NetworkError err = job->reply()->error();
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
if (err == QNetworkReply::OperationCanceledError && job->reply()->property(owncloudShouldSoftCancelPropertyName).isValid()) {
// Abort the job and try again later.
// This works around a bug in QNAM wich might reuse a non-empty buffer for the next request.
qDebug() << "Forcing job abort on HTTP connection reset with Qt < 5.4.2.";
_propagator->_anotherSyncNeeded = true;
abortWithError(SyncFileItem::SoftError, tr("Forcing job abort on HTTP connection reset with Qt < 5.4.2."));
return;
}
#endif
if (err != QNetworkReply::NoError) {
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if(checkForProblemsWithShared(_item->_httpErrorCode,
tr("The file was edited locally but is part of a read only share. "
"It is restored and your edit is in the conflict file."))) {
return;
}
QByteArray replyContent = job->reply()->readAll();
qDebug() << replyContent; // display the XML error in the debug
QString errorString = errorMessage(job->errorString(), replyContent);
if (job->reply()->hasRawHeader("OC-ErrorString")) {
errorString = job->reply()->rawHeader("OC-ErrorString");
}
if (_item->_httpErrorCode == 412) {
// Precondition Failed: Maybe the bad etag is in the database, we need to clear the
// parent folder etag so we won't read from DB next sync.
_propagator->_journal->avoidReadFromDbOnNextSync(_item->_file);
_propagator->_anotherSyncNeeded = true;
}
SyncFileItem::Status status = classifyError(err, _item->_httpErrorCode,
&_propagator->_anotherSyncNeeded);
abortWithError(status, errorString);
return;
}
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
// The server needs some time to process the request and provide us with a poll URL
if (_item->_httpErrorCode == 202) {
_finished = true;
QString path = QString::fromUtf8(job->reply()->rawHeader("OC-Finish-Poll"));
if (path.isEmpty()) {
done(SyncFileItem::NormalError, tr("Poll URL missing"));
return;
}
startPollJob(path);
return;
}
// Check the file again post upload.
// Two cases must be considered separately: If the upload is finished,
// the file is on the server and has a changed ETag. In that case,
// the etag has to be properly updated in the client journal, and because
// of that we can bail out here with an error. But we can reschedule a
// sync ASAP.
// But if the upload is ongoing, because not all chunks were uploaded
// yet, the upload can be stopped and an error can be displayed, because
// the server hasn't registered the new file yet.
QByteArray etag = getEtagFromReply(job->reply());
bool finished = etag.length() > 0;
// Check if the file still exists
const QString fullFilePath(_propagator->getFilePath(_item->_file));
if( !FileSystem::fileExists(fullFilePath) ) {
if (!finished) {
abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync."));
return;
} else {
_propagator->_anotherSyncNeeded = true;
}
}
// Check whether the file changed since discovery.
if (! FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) {
_propagator->_anotherSyncNeeded = true;
if( !finished ) {
abortWithError(SyncFileItem::SoftError, tr("Local file changed during sync."));
// FIXME: the legacy code was retrying for a few seconds.
// and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW
return;
}
}
if (!finished) {
// Proceed to next chunk.
if (_currentChunk >= _chunkCount) {
if (!_jobs.empty()) {
// just wait for the other job to finish.
return;
}
_finished = true;
done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag was present)"));
return;
}
// Deletes an existing blacklist entry on successful chunk upload
if (_item->_hasBlacklistEntry) {
_propagator->_journal->wipeErrorBlacklistEntry(_item->_file);
_item->_hasBlacklistEntry = false;
}
SyncJournalDb::UploadInfo pi;
pi._valid = true;
auto currentChunk = job->_chunk;
foreach (auto *job, _jobs) {
// Take the minimum finished one
if (auto putJob = qobject_cast<PUTFileJob*>(job)) {
currentChunk = qMin(currentChunk, putJob->_chunk - 1);
}
}
pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount ; // next chunk to start with
pi._transferid = _transferId;
pi._modtime = Utility::qDateTimeFromTime_t(_item->_modtime);
_propagator->_journal->setUploadInfo(_item->_file, pi);
_propagator->_journal->commit("Upload info");
startNextChunk();
return;
}
// the following code only happens after all chunks were uploaded.
_finished = true;
// the file id should only be empty for new files up- or downloaded
QByteArray fid = job->reply()->rawHeader("OC-FileID");
if( !fid.isEmpty() ) {
if( !_item->_fileId.isEmpty() && _item->_fileId != fid ) {
qDebug() << "WARN: File ID changed!" << _item->_fileId << fid;
}
_item->_fileId = fid;
}
_item->_etag = etag;
_item->_responseTimeStamp = job->responseTimestamp();
if (job->reply()->rawHeader("X-OC-MTime") != "accepted") {
// X-OC-MTime is supported since owncloud 5.0. But not when chunking.
// Normally Owncloud 6 always puts X-OC-MTime
qWarning() << "Server does not support X-OC-MTime" << job->reply()->rawHeader("X-OC-MTime");
// Well, the mtime was not set
done(SyncFileItem::SoftError, "Server does not support X-OC-MTime");
}
// performance logging
_item->_requestDuration = _stopWatch.stop();
qDebug() << "*==* duration UPLOAD" << _item->_size
<< _stopWatch.durationOfLap(QLatin1String("ContentChecksum"))
<< _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum"))
<< _item->_requestDuration;
// The job might stay alive for the whole sync, release this tiny bit of memory.
_stopWatch.reset();
finalize(*_item);
}
void PropagateUploadFile::finalize(const SyncFileItem &copy)
{
// Normally, copy == _item, but when it comes from the UpdateMTimeAndETagJob, we need to do
// some updates
_item->_etag = copy._etag;
_item->_fileId = copy._fileId;
_item->_requestDuration = _duration.elapsed();
_finished = true;
if (!_propagator->_journal->setFileRecord(SyncJournalFileRecord(*_item, _propagator->getFilePath(_item->_file)))) {
@ -807,92 +561,5 @@ void PropagateUploadFile::finalize(const SyncFileItem &copy)
done(SyncFileItem::Success);
}
void PropagateUploadFile::slotUploadProgress(qint64 sent, qint64 total)
{
// Completion is signaled with sent=0, total=0; avoid accidentally
// resetting progress due to the sent being zero by ignoring it.
// finishedSignal() is bound to be emitted soon anyway.
// See https://bugreports.qt.io/browse/QTBUG-44782.
if (sent == 0 && total == 0) {
return;
}
int progressChunk = _currentChunk + _startChunk - 1;
if (progressChunk >= _chunkCount)
progressChunk = _currentChunk - 1;
// amount is the number of bytes already sent by all the other chunks that were sent
// not including this one.
// FIXME: this assumes all chunks have the same size, which is true only if the last chunk
// has not been finished (which should not happen because the last chunk is sent sequentially)
quint64 amount = progressChunk * chunkSize();
sender()->setProperty("byteWritten", sent);
if (_jobs.count() > 1) {
amount -= (_jobs.count() -1) * chunkSize();
foreach (QObject *j, _jobs) {
amount += j->property("byteWritten").toULongLong();
}
} else {
// sender() is the only current job, no need to look at the byteWritten properties
amount += sent;
}
emit progress(*_item, amount);
}
void PropagateUploadFile::startPollJob(const QString& path)
{
PollJob* job = new PollJob(_propagator->account(), path, _item,
_propagator->_journal, _propagator->_localDir, this);
connect(job, SIGNAL(finishedSignal()), SLOT(slotPollFinished()));
SyncJournalDb::PollInfo info;
info._file = _item->_file;
info._url = path;
info._modtime = _item->_modtime;
_propagator->_journal->setPollInfo(info);
_propagator->_journal->commit("add poll info");
_propagator->_activeJobList.append(this);
job->start();
}
void PropagateUploadFile::slotPollFinished()
{
PollJob *job = qobject_cast<PollJob *>(sender());
Q_ASSERT(job);
_propagator->_activeJobList.removeOne(this);
if (job->_item->_status != SyncFileItem::Success) {
_finished = true;
done(job->_item->_status, job->_item->_errorString);
return;
}
finalize(*job->_item);
}
void PropagateUploadFile::slotJobDestroyed(QObject* job)
{
_jobs.erase(std::remove(_jobs.begin(), _jobs.end(), job) , _jobs.end());
}
void PropagateUploadFile::abort()
{
foreach(auto *job, _jobs) {
if (job->reply()) {
qDebug() << Q_FUNC_INFO << job << this->_item->_file;
job->reply()->abort();
}
}
}
// This function is used whenever there is an error occuring and jobs might be in progress
void PropagateUploadFile::abortWithError(SyncFileItem::Status status, const QString &error)
{
_finished = true;
abort();
done(status, error);
}
}

124
src/libsync/propagateupload.h
View file

@ -155,10 +155,87 @@ signals:
};
/**
* @brief The PropagateUploadFile class
* @brief The PropagateUploadFileCommon class is the code common between all chunking algorithms
* @ingroup libsync
*
* State Machine:
*
* +---> start() --> (delete job) -------+
* | |
* +--> slotComputeContentChecksum() <---+
* |
* v
* slotComputeTransmissionChecksum()
* |
* v
* slotStartUpload() -> doStartUpload()
* .
* .
* v
* finalize() or abortWithError() or startPollJob()
*/
class PropagateUploadFile : public PropagateItemJob {
class PropagateUploadFileCommon : public PropagateItemJob {
Q_OBJECT
protected:
QElapsedTimer _duration;
QVector<AbstractNetworkJob*> _jobs; /// network jobs that are currently in transit
bool _finished; /// Tells that all the jobs have been finished
bool _deleteExisting;
// measure the performance of checksum calc and upload
Utility::StopWatch _stopWatch;
QByteArray _transmissionChecksum;
QByteArray _transmissionChecksumType;
public:
PropagateUploadFileCommon(OwncloudPropagator* propagator,const SyncFileItemPtr& item)
: PropagateItemJob(propagator, item), _finished(false), _deleteExisting(false) {}
/**
* Whether an existing entity with the same name may be deleted before
* the upload.
*
* Default: false.
*/
void setDeleteExisting(bool enabled);
void start() Q_DECL_OVERRIDE;
private slots:
void slotComputeContentChecksum();
// Content checksum computed, compute the transmission checksum
void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum);
// transmission checksum computed, prepare the upload
void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
public:
virtual void doStartUpload() = 0;
void startPollJob(const QString& path);
void finalize();
void abortWithError(SyncFileItem::Status status, const QString &error);
public slots:
void abort() Q_DECL_OVERRIDE;
void slotJobDestroyed(QObject *job);
private slots:
void slotPollFinished();
protected:
// Bases headers that need to be sent with every chunk
QMap<QByteArray, QByteArray> headers();
};
/**
* @ingroup libsync
*
* Propagation job, impementing the old chunking agorithm
*
*/
class PropagateUploadFileV1 : public PropagateUploadFileCommon {
Q_OBJECT
private:
@ -176,48 +253,21 @@ private:
int _currentChunk;
int _chunkCount; /// Total number of chunks for this file
int _transferId; /// transfer id (part of the url)
QElapsedTimer _duration;
QVector<AbstractNetworkJob*> _jobs; /// network jobs that are currently in transit
bool _finished; // Tells that all the jobs have been finished
// measure the performance of checksum calc and upload
Utility::StopWatch _stopWatch;
QByteArray _transmissionChecksum;
QByteArray _transmissionChecksumType;
bool _deleteExisting;
quint64 chunkSize() const { return _propagator->chunkSize(); }
public:
PropagateUploadFile(OwncloudPropagator* propagator,const SyncFileItemPtr& item)
: PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false), _deleteExisting(false) {}
void start() Q_DECL_OVERRIDE;
/**
* Whether an existing entity with the same name may be deleted before
* the upload.
*
* Default: false.
*/
void setDeleteExisting(bool enabled);
public:
PropagateUploadFileV1(OwncloudPropagator* propagator,const SyncFileItemPtr& item) :
PropagateUploadFileCommon(propagator,item) {}
void doStartUpload() Q_DECL_OVERRIDE;
private slots:
void slotPutFinished();
void slotPollFinished();
void slotUploadProgress(qint64,qint64);
void abort() Q_DECL_OVERRIDE;
void startNextChunk();
void finalize(const SyncFileItem&);
void slotJobDestroyed(QObject *job);
void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum);
void slotComputeContentChecksum();
private:
void startPollJob(const QString& path);
void abortWithError(SyncFileItem::Status status, const QString &error);
void slotPutFinished();
void slotUploadProgress(qint64,qint64);
};
}

376
src/libsync/propagateuploadv1.cpp Normal file
View file

@ -0,0 +1,376 @@
/*
* Copyright (C) by Olivier Goffart <ogoffart@owncloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#include "config.h"
#include "propagateupload.h"
#include "owncloudpropagator_p.h"
#include "networkjobs.h"
#include "account.h"
#include "syncjournaldb.h"
#include "syncjournalfilerecord.h"
#include "utility.h"
#include "filesystem.h"
#include "propagatorjobs.h"
#include "checksums.h"
#include "syncengine.h"
#include "propagateremotedelete.h"
#include <json.h>
#include <QNetworkAccessManager>
#include <QFileInfo>
#include <QDir>
#include <cmath>
#include <cstring>
namespace OCC {
void PropagateUploadFileV1::doStartUpload()
{
_chunkCount = std::ceil(_item->_size / double(chunkSize()));
_startChunk = 0;
_transferId = qrand() ^ _item->_modtime ^ (_item->_size << 16);
const SyncJournalDb::UploadInfo progressInfo = _propagator->_journal->getUploadInfo(_item->_file);
if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item->_modtime ) {
_startChunk = progressInfo._chunk;
_transferId = progressInfo._transferid;
qDebug() << Q_FUNC_INFO << _item->_file << ": Resuming from chunk " << _startChunk;
}
_currentChunk = 0;
_duration.start();
emit progress(*_item, 0);
startNextChunk();
}
void PropagateUploadFileV1::startNextChunk()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
// We return now and when the _jobs are finished we will proceed with the last chunk
// NOTE: Some other parts of the code such as slotUploadProgress also assume that the last chunk
// is sent last.
return;
}
quint64 fileSize = _item->_size;
auto headers = PropagateUploadFileCommon::headers();
headers["OC-Total-Length"] = QByteArray::number(fileSize);
headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize()));
QString path = _item->_file;
UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager);
qint64 chunkStart = 0;
qint64 currentChunkSize = fileSize;
bool isFinalChunk = false;
if (_chunkCount > 1) {
int sendingChunk = (_currentChunk + _startChunk) % _chunkCount;
// XOR with chunk size to make sure everything goes well if chunk size changes between runs
uint transid = _transferId ^ chunkSize();
qDebug() << "Upload chunk" << sendingChunk << "of" << _chunkCount << "transferid(remote)=" << transid;
path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk);
headers["OC-Chunked"] = "1";
chunkStart = chunkSize() * quint64(sendingChunk);
currentChunkSize = chunkSize();
if (sendingChunk == _chunkCount - 1) { // last chunk
currentChunkSize = (fileSize % chunkSize());
if( currentChunkSize == 0 ) { // if the last chunk pretends to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
isFinalChunk = true;
}
} else {
// if there's only one chunk, it's the final one
isFinalChunk = true;
}
if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
headers[checkSumHeaderC] = makeChecksumHeader(
_transmissionChecksumType, _transmissionChecksum);
}
const QString fileName = _propagator->getFilePath(_item->_file);
if (! device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
qDebug() << "ERR: Could not prepare upload device: " << device->errorString();
// If the file is currently locked, we want to retry the sync
// when it becomes available again.
if (FileSystem::isFileLocked(fileName)) {
emit _propagator->seenLockedFile(fileName);
}
// Soft error because this is likely caused by the user modifying his files while syncing
abortWithError( SyncFileItem::SoftError, device->errorString() );
delete device;
return;
}
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk);
_jobs.append(job);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished()));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64)));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64)));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_propagator->_activeJobList.append(this);
_currentChunk++;
bool parallelChunkUpload = true;
QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK");
if (!env.isEmpty()) {
parallelChunkUpload = env != "false" && env != "0";
} else {
int versionNum = _propagator->account()->serverVersionInt();
if (versionNum < 0x080003) {
// Disable parallel chunk upload severs older than 8.0.3 to avoid too many
// internal sever errors (#2743, #2938)
parallelChunkUpload = false;
}
}
if (_currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
parallelChunkUpload = false;
}
if (parallelChunkUpload && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob())
&& _currentChunk < _chunkCount ) {
startNextChunk();
}
if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) {
emit ready();
}
}
void PropagateUploadFileV1::slotPutFinished()
{
PUTFileJob *job = qobject_cast<PUTFileJob *>(sender());
Q_ASSERT(job);
slotJobDestroyed(job); // remove it from the _jobs list
qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS"
<< job->reply()->error()
<< (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString())
<< job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute)
<< job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute);
_propagator->_activeJobList.removeOne(this);
if (_finished) {
// We have sent the finished signal already. We don't need to handle any remaining jobs
return;
}
QNetworkReply::NetworkError err = job->reply()->error();
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
if (err == QNetworkReply::OperationCanceledError && job->reply()->property(owncloudShouldSoftCancelPropertyName).isValid()) {
// Abort the job and try again later.
// This works around a bug in QNAM wich might reuse a non-empty buffer for the next request.
qDebug() << "Forcing job abort on HTTP connection reset with Qt < 5.4.2.";
_propagator->_anotherSyncNeeded = true;
abortWithError(SyncFileItem::SoftError, tr("Forcing job abort on HTTP connection reset with Qt < 5.4.2."));
return;
}
#endif
if (err != QNetworkReply::NoError) {
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if(checkForProblemsWithShared(_item->_httpErrorCode,
tr("The file was edited locally but is part of a read only share. "
"It is restored and your edit is in the conflict file."))) {
return;
}
QByteArray replyContent = job->reply()->readAll();
qDebug() << replyContent; // display the XML error in the debug
QString errorString = errorMessage(job->errorString(), replyContent);
if (job->reply()->hasRawHeader("OC-ErrorString")) {
errorString = job->reply()->rawHeader("OC-ErrorString");
}
if (_item->_httpErrorCode == 412) {
// Precondition Failed: Maybe the bad etag is in the database, we need to clear the
// parent folder etag so we won't read from DB next sync.
_propagator->_journal->avoidReadFromDbOnNextSync(_item->_file);
_propagator->_anotherSyncNeeded = true;
}
SyncFileItem::Status status = classifyError(err, _item->_httpErrorCode,
&_propagator->_anotherSyncNeeded);
abortWithError(status, errorString);
return;
}
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
// The server needs some time to process the request and provide us with a poll URL
if (_item->_httpErrorCode == 202) {
_finished = true;
QString path = QString::fromUtf8(job->reply()->rawHeader("OC-Finish-Poll"));
if (path.isEmpty()) {
done(SyncFileItem::NormalError, tr("Poll URL missing"));
return;
}
startPollJob(path);
return;
}
// Check the file again post upload.
// Two cases must be considered separately: If the upload is finished,
// the file is on the server and has a changed ETag. In that case,
// the etag has to be properly updated in the client journal, and because
// of that we can bail out here with an error. But we can reschedule a
// sync ASAP.
// But if the upload is ongoing, because not all chunks were uploaded
// yet, the upload can be stopped and an error can be displayed, because
// the server hasn't registered the new file yet.
QByteArray etag = getEtagFromReply(job->reply());
bool finished = etag.length() > 0;
// Check if the file still exists
const QString fullFilePath(_propagator->getFilePath(_item->_file));
if( !FileSystem::fileExists(fullFilePath) ) {
if (!finished) {
abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync."));
return;
} else {
_propagator->_anotherSyncNeeded = true;
}
}
// Check whether the file changed since discovery.
if (! FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) {
_propagator->_anotherSyncNeeded = true;
if( !finished ) {
abortWithError(SyncFileItem::SoftError, tr("Local file changed during sync."));
// FIXME: the legacy code was retrying for a few seconds.
// and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW
return;
}
}
if (!finished) {
// Proceed to next chunk.
if (_currentChunk >= _chunkCount) {
if (!_jobs.empty()) {
// just wait for the other job to finish.
return;
}
_finished = true;
done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag was present)"));
return;
}
// Deletes an existing blacklist entry on successful chunk upload
if (_item->_hasBlacklistEntry) {
_propagator->_journal->wipeErrorBlacklistEntry(_item->_file);
_item->_hasBlacklistEntry = false;
}
SyncJournalDb::UploadInfo pi;
pi._valid = true;
auto currentChunk = job->_chunk;
foreach (auto *job, _jobs) {
// Take the minimum finished one
if (auto putJob = qobject_cast<PUTFileJob*>(job)) {
currentChunk = qMin(currentChunk, putJob->_chunk - 1);
}
}
pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount ; // next chunk to start with
pi._transferid = _transferId;
pi._modtime = Utility::qDateTimeFromTime_t(_item->_modtime);
_propagator->_journal->setUploadInfo(_item->_file, pi);
_propagator->_journal->commit("Upload info");
startNextChunk();
return;
}
// the following code only happens after all chunks were uploaded.
_finished = true;
// the file id should only be empty for new files up- or downloaded
QByteArray fid = job->reply()->rawHeader("OC-FileID");
if( !fid.isEmpty() ) {
if( !_item->_fileId.isEmpty() && _item->_fileId != fid ) {
qDebug() << "WARN: File ID changed!" << _item->_fileId << fid;
}
_item->_fileId = fid;
}
_item->_etag = etag;
_item->_responseTimeStamp = job->responseTimestamp();
if (job->reply()->rawHeader("X-OC-MTime") != "accepted") {
// X-OC-MTime is supported since owncloud 5.0. But not when chunking.
// Normally Owncloud 6 always puts X-OC-MTime
qWarning() << "Server does not support X-OC-MTime" << job->reply()->rawHeader("X-OC-MTime");
// Well, the mtime was not set
done(SyncFileItem::SoftError, "Server does not support X-OC-MTime");
}
// performance logging
_item->_requestDuration = _stopWatch.stop();
qDebug() << "*==* duration UPLOAD" << _item->_size
<< _stopWatch.durationOfLap(QLatin1String("ContentChecksum"))
<< _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum"))
<< _item->_requestDuration;
// The job might stay alive for the whole sync, release this tiny bit of memory.
_stopWatch.reset();
finalize();
}
void PropagateUploadFileV1::slotUploadProgress(qint64 sent, qint64 total)
{
// Completion is signaled with sent=0, total=0; avoid accidentally
// resetting progress due to the sent being zero by ignoring it.
// finishedSignal() is bound to be emitted soon anyway.
// See https://bugreports.qt.io/browse/QTBUG-44782.
if (sent == 0 && total == 0) {
return;
}
int progressChunk = _currentChunk + _startChunk - 1;
if (progressChunk >= _chunkCount)
progressChunk = _currentChunk - 1;
// amount is the number of bytes already sent by all the other chunks that were sent
// not including this one.
// FIXME: this assumes all chunks have the same size, which is true only if the last chunk
// has not been finished (which should not happen because the last chunk is sent sequentially)
quint64 amount = progressChunk * chunkSize();
sender()->setProperty("byteWritten", sent);
if (_jobs.count() > 1) {
amount -= (_jobs.count() -1) * chunkSize();
foreach (QObject *j, _jobs) {
amount += j->property("byteWritten").toULongLong();
}
} else {
// sender() is the only current job, no need to look at the byteWritten properties
amount += sent;
}
emit progress(*_item, amount);
}
}