CSync & Propagator: Support a direct download URL

This is for server file backends that support sending a
direct URL.

csync/src/csync.c

@@ -442,6 +442,8 @@ static int _csync_treewalk_visitor(void *obj, void *data) {
       trav.rename_path  = cur->destpath;
       trav.etag         = cur->etag;
       trav.file_id      = cur->file_id;
+      trav.directDownloadUrl = cur->directDownloadUrl;
+      trav.directDownloadCookies = cur->directDownloadCookies;
       trav.inode        = cur->inode;
 
       trav.error_status = cur->error_status;
@@ -464,7 +466,7 @@ static int _csync_treewalk_visitor(void *obj, void *data) {
 
       rc = (*visitor)(&trav, twctx->userdata);
       cur->instruction = trav.instruction;
-      if (trav.etag != cur->etag) {
+      if (trav.etag != cur->etag) { // FIXME It would be nice to have this documented
           SAFE_FREE(cur->etag);
           cur->etag = c_strdup(trav.etag);
       }
@@ -912,6 +914,8 @@ int  csync_abort_requested(CSYNC *ctx)
 void csync_file_stat_free(csync_file_stat_t *st)
 {
   if (st) {
+    SAFE_FREE(st->directDownloadUrl);
+    SAFE_FREE(st->directDownloadCookies);
     SAFE_FREE(st->etag);
     SAFE_FREE(st->destpath);
     SAFE_FREE(st);

csync/src/csync.h

@@ -189,6 +189,8 @@ struct csync_tree_walk_file_s {
     const char *rename_path;
     const char *etag;
     const char *file_id;
+    char *directDownloadUrl;
+    char *directDownloadCookies;
     struct {
         int64_t     size;
         time_t      modtime;

csync/src/csync_owncloud.c

@@ -515,6 +515,8 @@ static void results(void *userdata,
     const char *resourcetype = NULL;
     const char *md5sum = NULL;
     const char *file_id = NULL;
+    const char *directDownloadUrl = NULL;
+    const char *directDownloadCookies = NULL;
     const ne_status *status = NULL;
     char *path = ne_path_unescape( uri->path );
 
@@ -540,6 +542,8 @@ static void results(void *userdata,
     resourcetype = ne_propset_value( set, &ls_props[2] );
     md5sum       = ne_propset_value( set, &ls_props[3] );
     file_id      = ne_propset_value( set, &ls_props[4] );
+    directDownloadUrl = ne_propset_value( set, &ls_props[5] );
+    directDownloadCookies = ne_propset_value( set, &ls_props[6] );
 
     newres->type = resr_normal;
     if( clength == NULL && resourcetype && strncmp( resourcetype, "<DAV:collection>", 16 ) == 0) {
@@ -563,6 +567,13 @@ static void results(void *userdata,
 
     csync_vio_set_file_id(newres->file_id, file_id);
 
+    if (directDownloadUrl) {
+        newres->directDownloadUrl = c_strdup(directDownloadUrl);
+    }
+    if (directDownloadCookies) {
+        newres->directDownloadCookies = c_strdup(directDownloadCookies);
+    }
+
     /* prepend the new resource to the result list */
     newres->next   = fetchCtx->list;
     fetchCtx->list = newres;

csync/src/csync_owncloud_private.h

@@ -125,6 +125,8 @@ static const ne_propname ls_props[] = {
     { "DAV:", "resourcetype" },
     { "DAV:", "getetag"},
     { "http://owncloud.org/ns", "id"},
+    { "http://owncloud.org/ns", "directDownloadUrl"},
+    { "http://owncloud.org/ns", "directDownloadCookies"},
     { NULL, NULL }
 };
 
@@ -140,6 +142,10 @@ typedef struct resource {
     time_t             modtime;
     char*              md5;
     char               file_id[FILE_ID_BUF_SIZE+1];
+    // Those two are optional from the server. We can use those URL to download the file directly
+    // without going through the ownCloud instance.
+    char *directDownloadUrl;
+    char *directDownloadCookies;
 
     struct resource    *next;
 } resource;

csync/src/csync_owncloud_recursive_propfind.c

@@ -99,6 +99,8 @@ static void propfind_results_recursive(void *userdata,
 {
     struct resource *newres = 0;
     const char *clength, *modtime, *file_id = NULL;
+    const char *directDownloadUrl = NULL;
+    const char *directDownloadCookies = NULL;
     const char *resourcetype = NULL;
     const char *md5sum = NULL;
     const ne_status *status = NULL;
@@ -109,6 +111,7 @@ static void propfind_results_recursive(void *userdata,
     int depth = 0;
     csync_owncloud_ctx_t *ctx = (csync_owncloud_ctx_t*) userdata;
 
+
     (void) status;
 
     if (!ctx->propfind_recursive_cache) {
@@ -127,6 +130,8 @@ static void propfind_results_recursive(void *userdata,
     resourcetype = ne_propset_value( set, &ls_props[2] );
     md5sum       = ne_propset_value( set, &ls_props[3] );
     file_id      = ne_propset_value( set, &ls_props[4] );
+    directDownloadUrl = ne_propset_value( set, &ls_props[5] );
+    directDownloadCookies = ne_propset_value( set, &ls_props[6] );
 
     newres->type = resr_normal;
     if( resourcetype && strncmp( resourcetype, "<DAV:collection>", 16 ) == 0) {
@@ -157,6 +162,13 @@ static void propfind_results_recursive(void *userdata,
     DEBUG_WEBDAV("propfind_results_recursive %s [%s] %s", newres->uri, newres->type == resr_collection ? "collection" : "file", newres->md5);
     */
 
+    if (directDownloadUrl) {
+        newres->directDownloadUrl = c_strdup(directDownloadUrl);
+    }
+    if (directDownloadCookies) {
+        newres->directDownloadCookies = c_strdup(directDownloadCookies);
+    }
+
     /* Create new item in rb tree */
     if (newres->type == resr_collection) {
         DEBUG_WEBDAV("propfind_results_recursive %s is a folder", newres->uri);

csync/src/csync_owncloud_util.c

@@ -312,6 +312,15 @@ void resourceToFileStat(csync_vio_file_stat_t *lfs, struct resource *res )
     }
 
     csync_vio_file_stat_set_file_id(lfs, res->file_id);
+
+    if (res->directDownloadUrl) {
+        lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL;
+        lfs->directDownloadUrl = c_strdup(res->directDownloadUrl);
+    }
+    if (res->directDownloadCookies) {
+        lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES;
+        lfs->directDownloadCookies = c_strdup(res->directDownloadCookies);
+    }
 }
 
 /* WebDAV does not deliver permissions. Set a default here. */
@@ -346,6 +355,12 @@ struct resource* resource_dup(struct resource* o) {
     if( o->md5 ) {
         r->md5 = c_strdup(o->md5);
     }
+    if (o->directDownloadUrl) {
+        r->directDownloadUrl = c_strdup(o->directDownloadUrl);
+    }
+    if (o->directDownloadCookies) {
+        r->directDownloadCookies = c_strdup(o->directDownloadCookies);
+    }
     r->next = o->next;
     csync_vio_set_file_id(r->file_id, o->file_id);
 
@@ -360,6 +375,8 @@ void resource_free(struct resource* o) {
         SAFE_FREE(old->uri);
         SAFE_FREE(old->name);
         SAFE_FREE(old->md5);
+        SAFE_FREE(old->directDownloadUrl);
+        SAFE_FREE(old->directDownloadCookies);
         SAFE_FREE(old);
     }
 }
@@ -381,6 +398,8 @@ void free_fetchCtx( struct listdir_context *ctx )
         SAFE_FREE(res->name);
         SAFE_FREE(res->md5);
         memset( res->file_id, 0, FILE_ID_BUF_SIZE+1 );
+        SAFE_FREE(res->directDownloadUrl);
+        SAFE_FREE(res->directDownloadCookies);
 
         newres = res->next;
         SAFE_FREE(res);

csync/src/csync_private.h

@@ -177,6 +177,9 @@ struct csync_file_stat_s {
   char *destpath;   /* for renames */
   const char *etag;
   char file_id[FILE_ID_BUF_SIZE+1];  /* the ownCloud file id is fixed width of 21 byte. */
+  char *directDownloadUrl;
+  char *directDownloadCookies;
+
   CSYNC_STATUS error_status;
 
   enum csync_instructions_e instruction; /* u32 */

csync/src/csync_update.c

@@ -369,6 +369,15 @@ out:
       st->etag  = c_strdup(fs->etag);
   }
   csync_vio_set_file_id(st->file_id, fs->file_id);
+  if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL) {
+      SAFE_FREE(st->directDownloadUrl);
+      st->directDownloadUrl = c_strdup(fs->directDownloadUrl);
+  }
+  if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES) {
+      SAFE_FREE(st->directDownloadCookies);
+      st->directDownloadCookies = c_strdup(fs->directDownloadCookies);
+  }
+
 
 fastout:  /* target if the file information is read from database into st */
   st->phash = h;

csync/src/vio/csync_vio.c

@@ -24,6 +24,7 @@
 
 #include <errno.h>
 #include <stdio.h>
+#include <assert.h>
 
 #include "csync_private.h"
 #include "csync_util.h"
@@ -107,6 +108,7 @@ int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) {
   switch(ctx->replica) {
     case REMOTE_REPLICA:
       CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "ERROR: Cannot call remote stat, not implemented");
+      assert(ctx->replica != REMOTE_REPLICA);
       break;
     case LOCAL_REPLICA:
       rc = csync_vio_local_stat(uri, buf);

csync/src/vio/csync_vio_file_stat.c

@@ -36,6 +36,8 @@ void csync_vio_file_stat_destroy(csync_vio_file_stat_t *file_stat) {
   if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_ETAG) {
     SAFE_FREE(file_stat->etag);
   }
+  SAFE_FREE(file_stat->directDownloadUrl);
+  SAFE_FREE(file_stat->directDownloadCookies);
   SAFE_FREE(file_stat->name);
   SAFE_FREE(file_stat);
 }

csync/src/vio/csync_vio_file_stat.h

@@ -73,7 +73,9 @@ enum csync_vio_file_stat_fields_e {
 //  CSYNC_VIO_FILE_STAT_FIELDS_UID = 1 << 15,
 //  CSYNC_VIO_FILE_STAT_FIELDS_GID = 1 << 16,
   CSYNC_VIO_FILE_STAT_FIELDS_ETAG = 1 << 17,
-  CSYNC_VIO_FILE_STAT_FIELDS_FILE_ID = 1 << 18
+  CSYNC_VIO_FILE_STAT_FIELDS_FILE_ID = 1 << 18,
+  CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL = 1 << 19,
+  CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES = 1 << 20
 };
 
 
@@ -81,6 +83,8 @@ struct csync_vio_file_stat_s {
   char *name;
   char *etag;
   char file_id[FILE_ID_BUF_SIZE+1];
+  char *directDownloadUrl;
+  char *directDownloadCookies;
 
   time_t atime;
   time_t mtime;

src/creds/tokencredentials.cpp

@@ -83,7 +83,12 @@ protected:
         QNetworkRequest req(request);
         req.setRawHeader(QByteArray("Authorization"), QByteArray("Basic ") + credHash);
         //qDebug() << "Request for " << req.url() << "with authorization" << QByteArray::fromBase64(credHash);
-        req.setRawHeader(QByteArray("Cookie"), _cred->_token.toLocal8Bit());
+
+        // Append token cookie
+        QList<QNetworkCookie> cookies = request.header(QNetworkRequest::CookieHeader).value<QList<QNetworkCookie> >();
+        cookies.append(QNetworkCookie::parseCookies(_cred->_token.toUtf8()));
+        req.setHeader(QNetworkRequest::CookieHeader, QVariant::fromValue(cookies));
+
         return MirallAccessManager::createRequest(op, req, outgoingData);
     }
 private:
@@ -112,7 +117,7 @@ void TokenCredentials::syncContextPreInit (CSYNC* ctx)
 
 void TokenCredentials::syncContextPreStart (CSYNC* ctx)
 {
-    csync_set_module_property(ctx, "session_key", _token.toLocal8Bit().data());
+    csync_set_module_property(ctx, "session_key", _token.toUtf8().data());
 }
 
 bool TokenCredentials::changed(AbstractCredentials* credentials) const

src/mirall/propagator_legacy.cpp

@@ -494,6 +494,10 @@ void PropagateDownloadFileLegacy::start()
         _propagator->_journal->commit("download file start");
     }
 
+    if (!_item._directDownloadUrl.isEmpty()) {
+        qDebug() << Q_FUNC_INFO << "Direct download URL" << _item._directDownloadUrl << "not supported with legacy propagator, will go via ownCloud server";
+    }
+
     /* actually do the request */
     int retry = 0;
 

src/mirall/propagator_qnam.cpp

@@ -358,13 +358,38 @@ void PropagateUploadFileQNAM::abort()
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
+// DOES NOT take owncership of the device.
+GETFileJob::GETFileJob(Account* account, const QString& path, QIODevice *device,
+                    const QMap<QByteArray, QByteArray> &headers, QByteArray expectedEtagForResume,
+                    QObject* parent)
+: AbstractNetworkJob(account, path, parent),
+  _device(device), _headers(headers), _expectedEtagForResume(expectedEtagForResume),
+  _errorStatus(SyncFileItem::NoStatus)
+{
+}
+
+GETFileJob::GETFileJob(Account* account, const QUrl& url, QIODevice *device,
+                    const QMap<QByteArray, QByteArray> &headers,
+                    QObject* parent)
+: AbstractNetworkJob(account, url.toEncoded(), parent),
+  _device(device), _headers(headers),
+  _errorStatus(SyncFileItem::NoStatus), _directDownloadUrl(url)
+{
+}
+
+
 void GETFileJob::start() {
     QNetworkRequest req;
     for(QMap<QByteArray, QByteArray>::const_iterator it = _headers.begin(); it != _headers.end(); ++it) {
         req.setRawHeader(it.key(), it.value());
     }
 
-    setReply(davRequest("GET", path(), req));
+    if (_directDownloadUrl.isEmpty()) {
+        setReply(davRequest("GET", path(), req));
+    } else {
+        // Use direct URL
+        setReply(davRequest("GET", _directDownloadUrl, req));
+    }
     setupConnections(reply());
 
     if( reply()->error() != QNetworkReply::NoError ) {
@@ -386,17 +411,21 @@ void GETFileJob::slotMetaDataChanged()
         return;
     }
 
-    QByteArray etag = parseEtag(reply()->rawHeader("Etag"));
+    _etag = parseEtag(reply()->rawHeader("Etag"));
-
+    if (!_directDownloadUrl.isEmpty() && !_etag.isEmpty()) {
-    if (etag.isEmpty()) {
+        qDebug() << Q_FUNC_INFO << "Direct download used, ignoring server ETag" << _etag;
+        _etag = QByteArray(); // reset received ETag
+    } else if (!_directDownloadUrl.isEmpty()) {
+        // All fine, ETag empty and directDownloadUrl used
+    } else if (_etag.isEmpty()) {
         qDebug() << Q_FUNC_INFO << "No E-Tag reply by server, considering it invalid";
         _errorString = tr("No E-Tag received from server, check Proxy/Gateway");
         _errorStatus = SyncFileItem::NormalError;
         reply()->abort();
         return;
-    } else if (!_expectedEtagForResume.isEmpty() && _expectedEtagForResume != etag) {
+    } else if (!_expectedEtagForResume.isEmpty() && _expectedEtagForResume != _etag) {
         qDebug() << Q_FUNC_INFO <<  "We received a different E-Tag for resuming!"
-                << _expectedEtagForResume << "vs" << etag;
+                << _expectedEtagForResume << "vs" << _etag;
         _errorString = tr("We received a different E-Tag for resuming. Retrying next time.");
         _errorStatus = SyncFileItem::NormalError;
         reply()->abort();
@@ -431,6 +460,13 @@ void GETFileJob::slotReadyRead()
     resetTimeout();
 }
 
+void GETFileJob::slotTimeout()
+{
+    _errorString =  tr("Connection Timeout");
+    _errorStatus = SyncFileItem::FatalError;
+    reply()->abort();
+}
+
 void PropagateDownloadFileQNAM::start()
 {
     if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
@@ -438,7 +474,7 @@ void PropagateDownloadFileQNAM::start()
 
     qDebug() << Q_FUNC_INFO << _item._file << _propagator->_activeJobs;
 
-    // do a case clash check.
+    // do a klaas' case clash check.
     if( _propagator->localFileNameClash(_item._file) ) {
         done( SyncFileItem::NormalError, tr("File %1 can not be downloaded because of a local file name clash!")
               .arg(QDir::toNativeSeparators(_item._file)) );
@@ -490,8 +526,6 @@ void PropagateDownloadFileQNAM::start()
 
 
     QMap<QByteArray, QByteArray> headers;
-    /* Allow compressed content by setting the header */
-    //headers["Accept-Encoding"] = "gzip";
 
     if (_tmpFile.size() > 0) {
         quint64 done = _tmpFile.size();
@@ -506,9 +540,22 @@ void PropagateDownloadFileQNAM::start()
         _startSize = done;
     }
 
-    _job = new GETFileJob(AccountManager::instance()->account(),
+    if (_item._directDownloadUrl.isEmpty()) {
-                          _propagator->_remoteFolder + _item._file,
+        // Normal job, download from oC instance
-                          &_tmpFile, headers, expectedEtagForResume);
+        _job = new GETFileJob(AccountManager::instance()->account(),
+                              _propagator->_remoteFolder + _item._file,
+                              &_tmpFile, headers, expectedEtagForResume);
+    } else {
+        // We were provided a direct URL, use that one
+        if (!_item._directDownloadCookies.isEmpty()) {
+            headers["Cookie"] = _item._directDownloadCookies.toUtf8();
+        }
+        QUrl url = QUrl::fromUserInput(_item._directDownloadUrl);
+        _job = new GETFileJob(AccountManager::instance()->account(),
+                              url,
+                              &_tmpFile, headers);
+        qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item._file << _item._directDownloadUrl;
+    }
     _job->setTimeout(_propagator->httpTimeout() * 1000);
     connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotGetFinished()));
     connect(_job, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotDownloadProgress(qint64,qint64)));
@@ -546,7 +593,11 @@ void PropagateDownloadFileQNAM::slotGetFinished()
         return;
     }
 
-    _item._etag = parseEtag(job->reply()->rawHeader("Etag"));
+    if (!job->etag().isEmpty()) {
+        // The etag will be empty if we used a direct download URL.
+        // (If it was really empty by the server, the GETFileJob will have errored
+        _item._etag = parseEtag(job->etag());
+    }
     _item._requestDuration = job->duration();
     _item._responseTimeStamp = job->responseTimestamp();
 
@@ -628,13 +679,4 @@ void PropagateDownloadFileQNAM::abort()
         _job->reply()->abort();
 }
 
-void GETFileJob::slotTimeout()
-{
-    _errorString =  tr("Connection Timeout");
-    _errorStatus = SyncFileItem::FatalError;
-    reply()->abort();
-}
-
-
-
 }

src/mirall/propagator_qnam.h

@@ -110,15 +110,18 @@ class GETFileJob : public AbstractNetworkJob {
     QString _errorString;
     QByteArray _expectedEtagForResume;
     SyncFileItem::Status _errorStatus;
+    QUrl _directDownloadUrl;
+    QByteArray _etag;
 public:
 
     // DOES NOT take owncership of the device.
     explicit GETFileJob(Account* account, const QString& path, QIODevice *device,
                         const QMap<QByteArray, QByteArray> &headers, QByteArray expectedEtagForResume,
-                        QObject* parent = 0)
+                        QObject* parent = 0);
-    : AbstractNetworkJob(account, path, parent),
+    // For directDownloadUrl:
-      _device(device), _headers(headers), _expectedEtagForResume(expectedEtagForResume),
+    explicit GETFileJob(Account* account, const QUrl& url, QIODevice *device,
-      _errorStatus(SyncFileItem::NoStatus) {}
+                        const QMap<QByteArray, QByteArray> &headers,
+                        QObject* parent = 0);
 
     virtual void start();
     virtual bool finished() {
@@ -134,6 +137,8 @@ public:
 
     virtual void slotTimeout();
 
+    QByteArray &etag() { return _etag; }
+
 
 signals:
     void finishedSignal();

src/mirall/syncengine.cpp

@@ -263,6 +263,12 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
     item._instruction = file->instruction;
     item._direction = SyncFileItem::None;
     item._fileId = file->file_id;
+    if (file->directDownloadUrl) {
+        item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl );
+    }
+    if (file->directDownloadCookies) {
+        item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies );
+    }
 
     // record the seen files to be able to clean the journal later
     _seenFiles[item._file] = QString();

src/mirall/syncfileitem.h

@@ -86,6 +86,8 @@ public:
     quint64              _inode;
     bool                 _should_update_etag;
     QByteArray           _fileId;
+    QString              _directDownloadUrl;
+    QString              _directDownloadCookies;
     bool                 _blacklistedInDb;
 
     // Variables usefull to report to the user