mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-10-27 06:45:49 +03:00
Code Issues Projects Releases Packages Wiki Activity

Download: Don't trigger too many concurrent hash computations

Browse source 
Previously the job would only become "active" when the downloads
started. That meant that arbitrarily many hash computations could be
queued at the same time.

Since the the file was opened during future creation this could lead to
a "too many open files" problem if there were lots of new-new conflicts.

To change this:
- Make PropagateDownload become active when computing a hash
  asynchronously.
- Make the computation future open the file only once it gets run. This
  will make it less likely for this problem to occur even if thousands
  of these futures are queued.

For #7372
This commit is contained in:
Christian Kamm 2019-08-12 08:23:40 +02:00 committed by Kevin Ottens
parent 8a5a185752
commit ea829f96ca
No known key found for this signature in database
GPG key ID: 074BBBCB8DECC9E2
2 changed files with 9 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/common/checksums.cpp
View file

@ -225,14 +225,6 @@ QByteArray ComputeChecksum::checksumType() const
void ComputeChecksum::start(const QString &filePath)
{
qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread";
// make_unique() would be more appropriate, but QtConcurrent requires
// copyable types.
auto file = std::make_shared<QFile>(filePath);
if (!file->open(QIODevice::ReadOnly)) {
qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading to compute a checksum" << file->errorString();
emit done(QByteArray(), QByteArray());
return;
}
connect(&_watcher, &QFutureWatcherBase::finished,
this, &ComputeChecksum::slotCalculationDone,
@ -241,8 +233,13 @@ void ComputeChecksum::start(const QString &filePath)
// Capturing "file" extends its lifetime to the lifetime of the new thread.
// Bug: The thread will keep running even if ComputeChecksum is deleted.
auto type = checksumType();
_watcher.setFuture(QtConcurrent::run([file, type]() {
return ComputeChecksum::computeNow(file.get(), type);
_watcher.setFuture(QtConcurrent::run([filePath, type]() {
QFile file(filePath);
if (!file.open(QIODevice::ReadOnly)) {
qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading to compute a checksum" << file.errorString();
return QByteArray();
}
return ComputeChecksum::computeNow(&file, type);
}));
}

2
src/libsync/propagatedownload.cpp
View file

@ -464,6 +464,7 @@ void PropagateDownloadFile::startAfterIsEncryptedIsChecked()
computeChecksum->setChecksumType(parseChecksumHeaderType(_item->_checksumHeader));
connect(computeChecksum, &ComputeChecksum::done,
this, &PropagateDownloadFile::conflictChecksumComputed);
propagator()->_activeJobList.append(this);
computeChecksum->start(propagator()->getFilePath(_item->_file));
return;
}
@ -473,6 +474,7 @@ void PropagateDownloadFile::startAfterIsEncryptedIsChecked()
void PropagateDownloadFile::conflictChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum)
{
propagator()->_activeJobList.removeOne(this);
if (makeChecksumHeader(checksumType, checksum) == _item->_checksumHeader) {
// No download necessary, just update fs and journal metadata
qCDebug(lcPropagateDownload) << _item->_file << "remote and local checksum match";