Upload: Read file chunks gradually #7226

Instead of all at once, to reduce peak memory use.

Changing UploadDevice in this way requires keeping the file open for the
duration of the upload. It also means changes to open(), seek(), close()
to ensure that uses of the device work right when a request needs to
be resent.
This commit is contained in:
Christian Kamm 2019-06-06 12:22:02 +02:00 committed by Kevin Ottens
parent a7852e3aba
commit 452ed56571
No known key found for this signature in database
GPG key ID: 074BBBCB8DECC9E2
4 changed files with 51 additions and 38 deletions

View file

@ -443,8 +443,11 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh
doStartUpload();
}
UploadDevice::UploadDevice(BandwidthManager *bwm)
: _read(0)
UploadDevice::UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm)
: _file(fileName)
, _start(start)
, _size(size)
, _read(0)
, _bandwidthManager(bwm)
, _bandwidthQuota(0)
, _readWithProgress(0)
@ -462,29 +465,28 @@ UploadDevice::~UploadDevice()
}
}
bool UploadDevice::prepareAndOpen(const QString &fileName, qint64 start, qint64 size)
bool UploadDevice::open(QIODevice::OpenMode mode)
{
_data.clear();
_read = 0;
if (mode & QIODevice::WriteOnly)
return false;
QFile file(fileName);
QString openError;
if (!FileSystem::openAndSeekFileSharedRead(&file, &openError, start)) {
if (!FileSystem::openAndSeekFileSharedRead(&_file, &openError, _start)) {
setErrorString(openError);
return false;
}
size = qBound(0ll, size, FileSystem::getSize(fileName) - start);
_data.resize(size);
auto read = file.read(_data.data(), size);
if (read != size) {
setErrorString(file.errorString());
return false;
}
_size = qBound(0ll, _size, FileSystem::getSize(_file.fileName()) - _start);
_read = 0;
return QIODevice::open(QIODevice::ReadOnly);
return QIODevice::open(mode);
}
void UploadDevice::close()
{
_file.close();
QIODevice::close();
}
qint64 UploadDevice::writeData(const char *, qint64)
{
@ -494,15 +496,15 @@ qint64 UploadDevice::writeData(const char *, qint64)
qint64 UploadDevice::readData(char *data, qint64 maxlen)
{
if (_data.size() - _read <= 0) {
if (_size - _read <= 0) {
// at end
if (_bandwidthManager) {
_bandwidthManager->unregisterUploadDevice(this);
}
return -1;
}
maxlen = qMin(maxlen, _data.size() - _read);
if (maxlen == 0) {
maxlen = qMin(maxlen, _size - _read);
if (maxlen <= 0) {
return 0;
}
if (isChoked()) {
@ -515,9 +517,14 @@ qint64 UploadDevice::readData(char *data, qint64 maxlen)
}
_bandwidthQuota -= maxlen;
}
std::memcpy(data, _data.data() + _read, maxlen);
_read += maxlen;
return maxlen;
auto c = _file.read(data, maxlen);
if (c < 0) {
setErrorString(_file.errorString());
return -1;
}
_read += c;
return c;
}
void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
@ -530,17 +537,17 @@ void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
bool UploadDevice::atEnd() const
{
return _read >= _data.size();
return _read >= _size;
}
qint64 UploadDevice::size() const
{
return _data.size();
return _size;
}
qint64 UploadDevice::bytesAvailable() const
{
return _data.size() - _read + QIODevice::bytesAvailable();
return _size - _read + QIODevice::bytesAvailable();
}
// random access, we can seek
@ -554,10 +561,11 @@ bool UploadDevice::seek(qint64 pos)
if (!QIODevice::seek(pos)) {
return false;
}
if (pos < 0 || pos > _data.size()) {
if (pos < 0 || pos > _size) {
return false;
}
_read = pos;
_file.seek(_start + pos);
return true;
}

View file

@ -36,11 +36,11 @@ class UploadDevice : public QIODevice
{
Q_OBJECT
public:
UploadDevice(BandwidthManager *bwm);
UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm);
~UploadDevice();
/** Reads the data from the file and opens the device */
bool prepareAndOpen(const QString &fileName, qint64 start, qint64 size);
bool open(QIODevice::OpenMode mode) override;
void close() override;
qint64 writeData(const char *, qint64) override;
qint64 readData(char *data, qint64 maxlen) override;
@ -59,10 +59,15 @@ public:
signals:
private:
// The file data
QByteArray _data;
// Position in the data
qint64 _read;
/// The local file to read data from
QFile _file;
/// Start of the file data to use
qint64 _start = 0;
/// Amount of file data after _start to use
qint64 _size = 0;
/// Position between _start and _start+_size
qint64 _read = 0;
// Bandwidth manager related
QPointer<BandwidthManager> _bandwidthManager;

View file

@ -316,10 +316,10 @@ void PropagateUploadFileNG::startNextChunk()
return;
}
auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
const QString fileName = _fileToUpload._path;
if (!device->prepareAndOpen(fileName, _sent, _currentChunkSize)) {
auto device = std::make_unique<UploadDevice>(
fileName, _currentChunk, _currentChunkSize, &propagator()->_bandwidthManager);
if (!device->open(QIODevice::ReadOnly)) {
qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
// If the file is currently locked, we want to retry the sync

View file

@ -90,7 +90,6 @@ void PropagateUploadFileV1::startNextChunk()
QString path = _fileToUpload._file;
auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
qint64 chunkStart = 0;
qint64 currentChunkSize = fileSize;
bool isFinalChunk = false;
@ -124,8 +123,9 @@ void PropagateUploadFileV1::startNextChunk()
}
const QString fileName = _fileToUpload._path;
qDebug() << "Trying to upload" << fileName;
if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
auto device = std::make_unique<UploadDevice>(
fileName, chunkStart, currentChunkSize, &propagator()->_bandwidthManager);
if (!device->open(QIODevice::ReadOnly)) {
qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
// If the file is currently locked, we want to retry the sync