Upload: Read file chunks gradually #7226

Instead of all at once, to reduce peak memory use.

Changing UploadDevice in this way requires keeping the file open for the
duration of the upload. It also means changes to open(), seek(), close()
to ensure that uses of the device work right when a request needs to
be resent.

src/libsync/propagateupload.cpp

@@ -443,8 +443,11 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh
     doStartUpload();
 }
 
-UploadDevice::UploadDevice(BandwidthManager *bwm)
+UploadDevice::UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm)
-    : _read(0)
+    : _file(fileName)
+    , _start(start)
+    , _size(size)
+    , _read(0)
     , _bandwidthManager(bwm)
     , _bandwidthQuota(0)
     , _readWithProgress(0)
@@ -462,30 +465,29 @@ UploadDevice::~UploadDevice()
     }
 }
 
-bool UploadDevice::prepareAndOpen(const QString &fileName, qint64 start, qint64 size)
+bool UploadDevice::open(QIODevice::OpenMode mode)
 {
-    _data.clear();
+    if (mode & QIODevice::WriteOnly)
-    _read = 0;
+        return false;
 
-    QFile file(fileName);
     QString openError;
-    if (!FileSystem::openAndSeekFileSharedRead(&file, &openError, start)) {
+    if (!FileSystem::openAndSeekFileSharedRead(&_file, &openError, _start)) {
         setErrorString(openError);
         return false;
     }
 
-    size = qBound(0ll, size, FileSystem::getSize(fileName) - start);
+    _size = qBound(0ll, _size, FileSystem::getSize(_file.fileName()) - _start);
-    _data.resize(size);
+    _read = 0;
-    auto read = file.read(_data.data(), size);
+
-    if (read != size) {
+    return QIODevice::open(mode);
-        setErrorString(file.errorString());
-        return false;
 }
 
-    return QIODevice::open(QIODevice::ReadOnly);
+void UploadDevice::close()
+{
+    _file.close();
+    QIODevice::close();
 }
 
-
 qint64 UploadDevice::writeData(const char *, qint64)
 {
     ASSERT(false, "write to read only device");
@@ -494,15 +496,15 @@ qint64 UploadDevice::writeData(const char *, qint64)
 
 qint64 UploadDevice::readData(char *data, qint64 maxlen)
 {
-    if (_data.size() - _read <= 0) {
+    if (_size - _read <= 0) {
         // at end
         if (_bandwidthManager) {
             _bandwidthManager->unregisterUploadDevice(this);
         }
         return -1;
     }
-    maxlen = qMin(maxlen, _data.size() - _read);
+    maxlen = qMin(maxlen, _size - _read);
-    if (maxlen == 0) {
+    if (maxlen <= 0) {
         return 0;
     }
     if (isChoked()) {
@@ -515,9 +517,14 @@ qint64 UploadDevice::readData(char *data, qint64 maxlen)
         }
         _bandwidthQuota -= maxlen;
     }
-    std::memcpy(data, _data.data() + _read, maxlen);
+
-    _read += maxlen;
+    auto c = _file.read(data, maxlen);
-    return maxlen;
+    if (c < 0) {
+        setErrorString(_file.errorString());
+        return -1;
+    }
+    _read += c;
+    return c;
 }
 
 void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
@@ -530,17 +537,17 @@ void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
 
 bool UploadDevice::atEnd() const
 {
-    return _read >= _data.size();
+    return _read >= _size;
 }
 
 qint64 UploadDevice::size() const
 {
-    return _data.size();
+    return _size;
 }
 
 qint64 UploadDevice::bytesAvailable() const
 {
-    return _data.size() - _read + QIODevice::bytesAvailable();
+    return _size - _read + QIODevice::bytesAvailable();
 }
 
 // random access, we can seek
@@ -554,10 +561,11 @@ bool UploadDevice::seek(qint64 pos)
     if (!QIODevice::seek(pos)) {
         return false;
     }
-    if (pos < 0 || pos > _data.size()) {
+    if (pos < 0 || pos > _size) {
         return false;
     }
     _read = pos;
+    _file.seek(_start + pos);
     return true;
 }
 

src/libsync/propagateupload.h

@@ -36,11 +36,11 @@ class UploadDevice : public QIODevice
 {
     Q_OBJECT
 public:
-    UploadDevice(BandwidthManager *bwm);
+    UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm);
     ~UploadDevice();
 
-    /** Reads the data from the file and opens the device */
+    bool open(QIODevice::OpenMode mode) override;
-    bool prepareAndOpen(const QString &fileName, qint64 start, qint64 size);
+    void close() override;
 
     qint64 writeData(const char *, qint64) override;
     qint64 readData(char *data, qint64 maxlen) override;
@@ -59,10 +59,15 @@ public:
 signals:
 
 private:
-    // The file data
+    /// The local file to read data from
-    QByteArray _data;
+    QFile _file;
-    // Position in the data
+
-    qint64 _read;
+    /// Start of the file data to use
+    qint64 _start = 0;
+    /// Amount of file data after _start to use
+    qint64 _size = 0;
+    /// Position between _start and _start+_size
+    qint64 _read = 0;
 
     // Bandwidth manager related
     QPointer<BandwidthManager> _bandwidthManager;

src/libsync/propagateuploadng.cpp

@@ -316,10 +316,10 @@ void PropagateUploadFileNG::startNextChunk()
         return;
     }
 
-    auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
     const QString fileName = _fileToUpload._path;
-
+    auto device = std::make_unique<UploadDevice>(
-    if (!device->prepareAndOpen(fileName, _sent, _currentChunkSize)) {
+            fileName, _currentChunk, _currentChunkSize, &propagator()->_bandwidthManager);
+    if (!device->open(QIODevice::ReadOnly)) {
         qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
 
         // If the file is currently locked, we want to retry the sync

src/libsync/propagateuploadv1.cpp

@@ -90,7 +90,6 @@ void PropagateUploadFileV1::startNextChunk()
 
     QString path = _fileToUpload._file;
 
-    auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
     qint64 chunkStart = 0;
     qint64 currentChunkSize = fileSize;
     bool isFinalChunk = false;
@@ -124,8 +123,9 @@ void PropagateUploadFileV1::startNextChunk()
     }
 
     const QString fileName = _fileToUpload._path;
-    qDebug() << "Trying to upload" << fileName;
+    auto device = std::make_unique<UploadDevice>(
-    if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
+            fileName, chunkStart, currentChunkSize, &propagator()->_bandwidthManager);
+    if (!device->open(QIODevice::ReadOnly)) {
         qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
 
         // If the file is currently locked, we want to retry the sync