mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-10-23 21:05:44 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #5480 from owncloud/chunking-ng-cleanup

Browse source 
Chunking ng cleanup
This commit is contained in:
Olivier Goffart 2017-01-25 19:19:48 +01:00 committed by GitHub
commit 268fc97a71
6 changed files with 250 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/libsync/propagateuploadng.cpp
View file

@ -98,6 +98,12 @@ void PropagateUploadFileNG::doStartUpload()
this, SLOT(slotPropfindIterate(QString,QMap<QString,QString>)));
job->start();
return;
} else if (progressInfo._valid) {
// The upload info is stale. remove the stale chunks on the server
_transferId = progressInfo._transferid;
// Fire and forget. Any error will be ignored.
(new DeleteJob(propagator()->account(), chunkUrl(), this))->start();
// startNewUpload will reset the _transferId and the UploadInfo in the db.
}
startNewUpload();

12
src/libsync/syncengine.cpp
View file

@ -23,6 +23,7 @@
#include "syncfilestatus.h"
#include "csync_private.h"
#include "filesystem.h"
#include "propagateremotedelete.h"
#ifdef Q_OS_WIN
#include <windows.h>
@ -301,7 +302,16 @@ void SyncEngine::deleteStaleUploadInfos()
}
// Delete from journal.
_journal->deleteStaleUploadInfos(upload_file_paths);
auto ids = _journal->deleteStaleUploadInfos(upload_file_paths);
// Delete the stales chunk on the server.
if (account()->capabilities().chunkingNg()) {
foreach (uint transferId, ids) {
QUrl url = Utility::concatUrlPath(account()->url(), QLatin1String("remote.php/dav/uploads/")
+ account()->davUser() + QLatin1Char('/') + QString::number(transferId));
(new DeleteJob(account(), url, this))->start();
}
}
}
void SyncEngine::deleteStaleErrorBlacklistEntries()

13
src/libsync/syncjournaldb.cpp
View file

@ -1370,21 +1370,22 @@ void SyncJournalDb::setUploadInfo(const QString& file, const SyncJournalDb::Uplo
}
}
bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
QVector<uint> SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
{
QMutexLocker locker(&_mutex);
QVector<uint> ids;
if (!checkConnect()) {
return false;
return ids;
}
SqlQuery query(_db);
query.prepare("SELECT path FROM uploadinfo");
query.prepare("SELECT path,transferid FROM uploadinfo");
if (!query.exec()) {
QString err = query.error();
qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
return false;
return ids;
}
QStringList superfluousPaths;
@ -1393,10 +1394,12 @@ bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
const QString file = query.stringValue(0);
if (!keep.contains(file)) {
superfluousPaths.append(file);
ids.append(query.intValue(1));
}
}
return deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo");
deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo");
return ids;
}
SyncJournalErrorBlacklistRecord SyncJournalDb::errorBlacklistEntry( const QString& file )

3
src/libsync/syncjournaldb.h
View file

@ -105,7 +105,8 @@ public:
UploadInfo getUploadInfo(const QString &file);
void setUploadInfo(const QString &file, const UploadInfo &i);
bool deleteStaleUploadInfos(const QSet<QString>& keep);
// Return the list of transfer ids that were removed.
QVector<uint> deleteStaleUploadInfos(const QSet<QString>& keep);
SyncJournalErrorBlacklistRecord errorBlacklistEntry( const QString& );
bool deleteStaleErrorBlacklistEntries(const QSet<QString>& keep);

45
test/syncenginetestutils.h
View file

@ -315,7 +315,10 @@ public:
QString fileName = getFilePathFromUrl(request.url());
Q_ASSERT(!fileName.isNull()); // for root, it should be empty
const FileInfo *fileInfo = remoteRootFileInfo.find(fileName);
Q_ASSERT(fileInfo);
if (!fileInfo) {
QMetaObject::invokeMethod(this, "respond404", Qt::QueuedConnection);
return;
}
QString prefix = request.url().path().left(request.url().path().size() - fileName.size());
// Don't care about the request and just return a full propfind
@ -375,6 +378,13 @@ public:
emit finished();
}
Q_INVOKABLE void respond404() {
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 404);
setError(InternalServerError, "Not Found");
emit metaDataChanged();
emit finished();
}
void abort() override { }
qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
@ -524,7 +534,8 @@ class FakeGetReply : public QNetworkReply
Q_OBJECT
public:
const FileInfo *fileInfo;
QByteArray payload;
char payload;
int size;
FakeGetReply(FileInfo &remoteRootFileInfo, QNetworkAccessManager::Operation op, const QNetworkRequest &request, QObject *parent)
: QNetworkReply{parent} {
@ -540,8 +551,9 @@ public:
}
Q_INVOKABLE void respond() {
payload.fill(fileInfo->contentChar, fileInfo->size);
setHeader(QNetworkRequest::ContentLengthHeader, payload.size());
payload = fileInfo->contentChar;
size = fileInfo->size;
setHeader(QNetworkRequest::ContentLengthHeader, size);
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 200);
setRawHeader("OC-ETag", fileInfo->etag.toLatin1());
setRawHeader("ETag", fileInfo->etag.toLatin1());
@ -553,12 +565,12 @@ public:
}
void abort() override { }
qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
qint64 bytesAvailable() const override { return size + QIODevice::bytesAvailable(); }
qint64 readData(char *data, qint64 maxlen) override {
qint64 len = std::min(qint64{payload.size()}, maxlen);
strncpy(data, payload.constData(), len);
payload.remove(0, len);
qint64 len = std::min(qint64{size}, maxlen);
std::fill_n(data, len, payload);
size -= len;
return len;
}
};
@ -586,7 +598,7 @@ public:
Q_ASSERT(sourceFolder->isDir);
int count = 0;
int size = 0;
char payload = '*';
char payload = '\0';
do {
QString chunkName = QString::number(count).rightJustified(8, '0');
@ -596,6 +608,7 @@ public:
Q_ASSERT(!x.isDir);
Q_ASSERT(x.size > 0); // There should not be empty chunks
size += x.size;
Q_ASSERT(!payload || payload == x.contentChar);
payload = x.contentChar;
++count;
} while(true);
@ -607,7 +620,12 @@ public:
Q_ASSERT(!fileName.isEmpty());
if ((fileInfo = remoteRootFileInfo.find(fileName))) {
QCOMPARE(request.rawHeader("If"), QByteArray("<" + request.rawHeader("Destination") + "> ([\"" + fileInfo->etag.toLatin1() + "\"])"));
QVERIFY(request.hasRawHeader("If")); // The client should put this header
if (request.rawHeader("If") != QByteArray("<" + request.rawHeader("Destination") +
"> ([\"" + fileInfo->etag.toLatin1() + "\"])")) {
QMetaObject::invokeMethod(this, "respondPreconditionFailed", Qt::QueuedConnection);
return;
}
fileInfo->size = size;
fileInfo->contentChar = payload;
} else {
@ -632,6 +650,13 @@ public:
emit finished();
}
Q_INVOKABLE void respondPreconditionFailed() {
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 412);
setError(InternalServerError, "Precondition Failed");
emit metaDataChanged();
emit finished();
}
void abort() override { }
qint64 readData(char *, qint64) override { return 0; }
};

211
test/testchunkingng.cpp
View file

@ -11,6 +11,36 @@
using namespace OCC;
/* Upload a 1/3 of a file of given size.
* fakeFolder needs to be synchronized */
static void partialUpload(FakeFolder &fakeFolder, const QString &name, int size)
{
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.uploadState().children.count(), 0); // The state should be clean
fakeFolder.localModifier().insert(name, size);
// Abort when the upload is at 1/3
int sizeWhenAbort = -1;
auto con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
[&](const ProgressInfo &progress) {
if (progress.completedSize() > (progress.totalSize() /3 )) {
sizeWhenAbort = progress.completedSize();
fakeFolder.syncEngine().abort();
}
});
QVERIFY(!fakeFolder.syncOnce()); // there should have been an error
QObject::disconnect(con);
QVERIFY(sizeWhenAbort > 0);
QVERIFY(sizeWhenAbort < size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1); // the transfer was done with chunking
auto upStateChildren = fakeFolder.uploadState().children.first().children;
QCOMPARE(sizeWhenAbort, std::accumulate(upStateChildren.cbegin(), upStateChildren.cend(), 0,
[](int s, const FileInfo &i) { return s + i.size; }));
}
class TestChunkingNG : public QObject
{
Q_OBJECT
@ -40,38 +70,173 @@ private slots:
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 300 * 1000 * 1000; // 300 MB
fakeFolder.localModifier().insert("A/a0", size);
// Abort when the upload is at 1/3
int sizeWhenAbort = -1;
auto con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
[&](const ProgressInfo &progress) {
if (progress.completedSize() > (progress.totalSize() /3 )) {
sizeWhenAbort = progress.completedSize();
fakeFolder.syncEngine().abort();
}
});
QVERIFY(!fakeFolder.syncOnce()); // there should have been an error
QObject::disconnect(con);
QVERIFY(sizeWhenAbort > 0);
QVERIFY(sizeWhenAbort < size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1); // the transfer was done with chunking
auto upStateChildren = fakeFolder.uploadState().children.first().children;
QCOMPARE(sizeWhenAbort, std::accumulate(upStateChildren.cbegin(), upStateChildren.cend(), 0,
[](int s, const FileInfo &i) { return s + i.size; }));
partialUpload(fakeFolder, "A/a0", size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
auto chunkingId = fakeFolder.uploadState().children.first().name;
// Add a fake file to make sure it gets deleted
fakeFolder.uploadState().children.first().insert("10000", size);
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
// The same chunk id was re-used
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
}
// We modify the file locally after it has been partially uploaded
void testRemoveStale1() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 300 * 1000 * 1000; // 300 MB
partialUpload(fakeFolder, "A/a0", size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
auto chunkingId = fakeFolder.uploadState().children.first().name;
fakeFolder.localModifier().setContents("A/a0", 'B');
fakeFolder.localModifier().appendByte("A/a0");
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.uploadState().children.count(), 1); // The same chunk id was re-used
QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size + 1);
// A different chunk id was used, and the previous one is removed
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
}
// We remove the file locally after it has been partially uploaded
void testRemoveStale2() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 300 * 1000 * 1000; // 300 MB
partialUpload(fakeFolder, "A/a0", size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
fakeFolder.localModifier().remove("A/a0");
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.uploadState().children.count(), 0);
}
void testCreateConflictWhileSyncing() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 150 * 1000 * 1000; // 150 MB
// Put a file on the server and download it.
fakeFolder.remoteModifier().insert("A/a0", size);
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
// Modify the file localy and start the upload
fakeFolder.localModifier().setContents("A/a0", 'B');
fakeFolder.localModifier().appendByte("A/a0");
// But in the middle of the sync, modify the file on the server
QMetaObject::Connection con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
[&](const ProgressInfo &progress) {
if (progress.completedSize() > (progress.totalSize() / 2 )) {
fakeFolder.remoteModifier().setContents("A/a0", 'C');
QObject::disconnect(con);
}
});
QVERIFY(!fakeFolder.syncOnce());
// There was a precondition failed error, this means wen need to sync again
QCOMPARE(fakeFolder.syncEngine().isAnotherSyncNeeded(), ImmediateFollowUp);
QCOMPARE(fakeFolder.uploadState().children.count(), 1); // We did not clean the chunks at this point
// Now we will download the server file and create a conflict
QVERIFY(fakeFolder.syncOnce());
auto localState = fakeFolder.currentLocalState();
// A0 is the one from the server
QCOMPARE(localState.find("A/a0")->size, size);
QCOMPARE(localState.find("A/a0")->contentChar, 'C');
// There is a conflict file with our version
auto &stateAChildren = localState.find("A")->children;
auto it = std::find_if(stateAChildren.cbegin(), stateAChildren.cend(), [&](const FileInfo &fi) {
return fi.name.startsWith("a0_conflict");
});
QVERIFY(it != stateAChildren.cend());
QCOMPARE(it->contentChar, 'B');
QCOMPARE(it->size, size+1);
// Remove the conflict file so the comparison works!
fakeFolder.localModifier().remove("A/" + it->name);
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.uploadState().children.count(), 0); // The last sync cleaned the chunks
}
void testModifyLocalFileWhileUploading() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 150 * 1000 * 1000; // 150 MB
fakeFolder.localModifier().insert("A/a0", size);
// middle of the sync, modify the file
QMetaObject::Connection con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
[&](const ProgressInfo &progress) {
if (progress.completedSize() > (progress.totalSize() / 2 )) {
fakeFolder.localModifier().setContents("A/a0", 'B');
fakeFolder.localModifier().appendByte("A/a0");
QObject::disconnect(con);
}
});
QVERIFY(!fakeFolder.syncOnce());
// There should be a followup sync
QCOMPARE(fakeFolder.syncEngine().isAnotherSyncNeeded(), ImmediateFollowUp);
QCOMPARE(fakeFolder.uploadState().children.count(), 1); // We did not clean the chunks at this point
auto chunkingId = fakeFolder.uploadState().children.first().name;
// Now we make a new sync which should upload the file for good.
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size+1);
// A different chunk id was used, and the previous one is removed
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
}
void testResumeServerDeletedChunks() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
const int size = 300 * 1000 * 1000; // 300 MB
partialUpload(fakeFolder, "A/a0", size);
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
auto chunkingId = fakeFolder.uploadState().children.first().name;
// Delete the chunks on the server
fakeFolder.uploadState().children.clear();
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
// A different chunk id was used
QCOMPARE(fakeFolder.uploadState().children.count(), 1);
QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
}
};
QTEST_GUILESS_MAIN(TestChunkingNG)