propagator-ng: be able to resume download

src/CMakeLists.txt

@@ -74,6 +74,7 @@ set(libsync_SRCS
     mirall/owncloudfolder.cpp
     mirall/csyncthread.cpp
     mirall/owncloudpropagator.cpp
+    mirall/progressdatabase.cpp
     mirall/fileutils.cpp
     mirall/theme.cpp
     mirall/owncloudtheme.cpp

src/mirall/csyncthread.cpp

@@ -19,6 +19,7 @@
 #include "mirall/logger.h"
 #include "mirall/owncloudinfo.h"
 #include "owncloudpropagator.h"
+#include "progressdatabase.h"
 
 #ifdef Q_OS_WIN
 #include <windows.h>
@@ -188,6 +189,7 @@ int CSyncThread::treewalkFile( TREE_WALK_FILE *file, bool remote )
     item._dir = SyncFileItem::None;
     item._isDirectory = file->type == CSYNC_FTW_TYPE_DIR;
     item._modtime = file->modtime;
+    item._etag = file->md5;
 
     SyncFileItem::Direction dir;
 
@@ -370,8 +372,11 @@ void CSyncThread::startSync()
     csync_set_module_property(_csync_ctx, "get_dav_session", &session);
     Q_ASSERT(session);
 
+    ProgressDatabase db;
+    db.load(_localPath);
+
     QString lastDeleted;
-    OwncloudPropagator propagator(session, _localPath, _remotePath);
+    OwncloudPropagator propagator(session, _localPath, _remotePath, &db);
     foreach (const SyncFileItem &item , _syncedItems) {
         Action a;
         if (!lastDeleted.isEmpty() && item._file.startsWith(lastDeleted)
@@ -428,6 +433,8 @@ void CSyncThread::startSync()
 //         return;
 //     }
 
+    db.save(_localPath);
+
     if( walkOk ) {
         if( csync_walk_local_tree(_csync_ctx, &walkFinalize, 0) < 0 ||
             csync_walk_remote_tree( _csync_ctx, &walkFinalize, 0 ) < 0 ) {

src/mirall/owncloudpropagator.cpp

@@ -14,6 +14,7 @@
  */
 
 #include "owncloudpropagator.h"
+#include "progressdatabase.h"
 #include <httpbf.h>
 #include <qfile.h>
 #include <qdir.h>
@@ -363,19 +364,41 @@ public:
 
 csync_instructions_e OwncloudPropagator::downloadFile(const SyncFileItem &item, bool isConflict)
 {
-    QTemporaryFile tmpFile(_localDir + item._file);
-    if (!tmpFile.open()) {
+    QString tmpFileName;
+    const ProgressDatabase::DownloadInfo* progressInfo = _progressDb->getDownloadInfo(item._file);
+    if (progressInfo) {
+        if (progressInfo->etag != item._etag) {
+            QFile::remove(_localDir + progressInfo->tmpfile);
+        } else {
+            tmpFileName = progressInfo->tmpfile;
+        }
+        _progressDb->remove(item._file);
+    }
+    if (tmpFileName.isEmpty()) {
+        tmpFileName = item._file + "." + QString::number(uint(qrand()), 16);
+    }
+
+    QFile tmpFile(_localDir + tmpFileName);
+    if (!tmpFile.open(QIODevice::Append)) {
         _errorString = tmpFile.errorString();
         _errorCode = CSYNC_ERR_FILESYSTEM;
         return CSYNC_INSTRUCTION_ERROR;
     }
 
+    //TODO: make tmpFile hidden and excluded in case it is still there in the next sync.
+
+    {
+        ProgressDatabase::DownloadInfo pi;
+        pi.etag = item._etag;
+        pi.tmpfile = tmpFileName;
+        _progressDb->setDownloadInfo(item._file, pi);
+        _progressDb->save(_localDir);
+    }
+
     /* actually do the request */
     int retry = 0;
-//     if (_progresscb) {
+
 //         ne_set_notifier(dav_session.ctx, ne_notify_status_cb, write_ctx);
-//         _progresscb(write_ctx->url, CSYNC_NOTIFY_START_DOWNLOAD, 0 , 0, dav_session.userdata);
-//     }
 
     QScopedPointer<char, QScopedPointerPodDeleter> uri(ne_path_escape((_remoteDir + item._file).toUtf8()));
     DownloadContext writeCtx(&tmpFile);
@@ -411,6 +434,12 @@ csync_instructions_e OwncloudPropagator::downloadFile(const SyncFileItem &item,
 
         if( updateErrorFromSession(neon_stat, req.data() ) ) {
             qDebug("Error GET: Neon: %d", neon_stat);
+            if (tmpFile.size() == 0) {
+                // don't keep the temporary file if it is empty.
+                tmpFile.close();
+                tmpFile.remove();
+                _progressDb->remove(item._file);
+            }
             return CSYNC_INSTRUCTION_ERROR;
         }
 
@@ -428,6 +457,8 @@ csync_instructions_e OwncloudPropagator::downloadFile(const SyncFileItem &item,
 
         // compare the files to see if there was an actual conflict.
         if (fileEquals(fn, tmpFile.fileName())) {
+            tmpFile.remove();
+            _progressDb->remove(item._file);
             return CSYNC_INSTRUCTION_UPDATED;
         }
 
@@ -467,7 +498,7 @@ csync_instructions_e OwncloudPropagator::downloadFile(const SyncFileItem &item,
     }
 #endif
 
-    tmpFile.setAutoRemove(false);
+    _progressDb->remove(item._file);
 
     struct timeval times[2];
     times[0].tv_sec = times[1].tv_sec = item._modtime;

src/mirall/owncloudpropagator.h

@@ -24,10 +24,13 @@ struct ne_decompress_s;
 
 namespace Mirall {
 
+class ProgressDatabase;
+
 class OwncloudPropagator {
     QString _localDir; // absolute path to the local directory. ends with '/'
     QString _remoteDir; // path to the root of the remote. ends with '/'
     ne_session_s *_session;
+    ProgressDatabase *_progressDb;
 
     bool check_neon_session();
 
@@ -47,11 +50,13 @@ class OwncloudPropagator {
 
 
 public:
-    OwncloudPropagator(ne_session_s *session, const QString &localDir, const QString &remoteDir)
+    OwncloudPropagator(ne_session_s *session, const QString &localDir, const QString &remoteDir,
+                       ProgressDatabase *progressDb)
             : _session(session)
             , _errorCode(CSYNC_ERR_NONE)
             , _localDir(localDir)
-            , _remoteDir(remoteDir) {
+            , _remoteDir(remoteDir)
+            , _progressDb(progressDb) {
         if (!localDir.endsWith(QChar('/'))) _localDir+='/';
         if (!remoteDir.endsWith(QChar('/'))) _remoteDir+='/';
     }

src/mirall/progressdatabase.cpp

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) by Olivier Goffart <ogoffart@owncloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "progressdatabase.h"
+#include <QFile>
+
+namespace Mirall {
+
+QDataStream& operator<<(QDataStream&d , const ProgressDatabase::DownloadInfo &i)
+{ return d << i.tmpfile << i.etag; }
+QDataStream& operator>>(QDataStream&d , ProgressDatabase::DownloadInfo &i)
+{ return d >> i.tmpfile >> i.etag;  }
+QDataStream& operator<<(QDataStream&d , const ProgressDatabase::UploadInfo &i)
+{ return d << i.chunk << i.transferid << i.size << qlonglong(i.mtime); }
+QDataStream& operator>>(QDataStream&d , ProgressDatabase::UploadInfo &i) {
+    qlonglong mtime;
+    d >> i.chunk >> i.transferid >> i.size >> mtime;
+    i.mtime = mtime;
+    return d;
+}
+
+
+void ProgressDatabase::load(const QString& rootDirectory)
+{
+    QMutexLocker locker(&_mutex);
+    QFile f(rootDirectory + "/.csync-progressdatabase");
+    if (!f.open(QIODevice::ReadOnly)) {
+        return;
+    }
+    QDataStream stream(&f);
+
+    QByteArray magic;
+    stream >> magic;
+    if (magic != "csyncpdb_1")
+        return;
+
+    stream >> _down >> _up;
+}
+
+void ProgressDatabase::save(const QString& rootDirectory)
+{
+    QMutexLocker locker(&_mutex);
+    QFile f(rootDirectory + "/.csync-progressdatabase");
+    if (!f.open(QIODevice::WriteOnly)) {
+        return;
+    }
+    QDataStream stream(&f);
+    stream << QByteArray("csyncpdb_1") << _down << _up;
+}
+
+
+
+
+
+}

src/mirall/progressdatabase.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) by Olivier Goffart <ogoffart@owncloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef PROGRESSDATABASE_H
+#define PROGRESSDATABASE_H
+
+#include <QString>
+#include <QList>
+#include <QHash>
+#include <QMutex>
+
+
+namespace Mirall {
+
+class ProgressDatabase {
+public:
+    struct DownloadInfo {
+        QString tmpfile;
+        QByteArray etag;
+    };
+    typedef QHash<QString, DownloadInfo > DownloadInfoHash;
+    struct UploadInfo {
+        int chunk;
+        int transferid;
+        quint64 size;
+        time_t mtime;
+    };
+    typedef QHash<QString, UploadInfo > UploadInfoHash;
+
+    void load(const QString &rootDirectory);
+    void save(const QString &rootDirectory);
+
+    const DownloadInfo *getDownloadInfo(const QString &file) const {
+        QMutexLocker locker(&_mutex);
+        DownloadInfoHash::const_iterator it = _down.constFind(file);
+        if (it == _down.end())
+            return 0;
+        return &it.value();
+    }
+
+    const UploadInfo *getUploadInfo(const QString &file) const {
+        QMutexLocker locker(&_mutex);
+        UploadInfoHash::const_iterator it = _up.constFind(file);
+        if (it == _up.end())
+            return 0;
+        return &it.value();
+    }
+
+    void setDownloadInfo(const QString &file, const DownloadInfo &i) {
+        QMutexLocker locker(&_mutex);
+        _down[file] = i;
+    }
+
+    void setUploadInfo(const QString &file, const UploadInfo &i) {
+        QMutexLocker locker(&_mutex);
+        _up[file] = i;
+    }
+
+    void remove(const QString &file) {
+        QMutexLocker locker(&_mutex);
+        _down.remove(file);
+        _up.remove(file);
+    }
+
+private:
+    DownloadInfoHash _down;
+    UploadInfoHash _up;
+    QString _rootPath;
+    mutable QMutex _mutex;
+};
+
+
+}
+
+#endif

src/mirall/syncfileitem.h

@@ -37,6 +37,7 @@ public:
     Direction _dir;
     bool _isDirectory;
     time_t _modtime;
+    QByteArray _etag;
 
     QString _errorString;
     QString _errorDetail;