mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-11-25 22:46:04 +03:00
Code Issues Projects Releases Packages Wiki Activity

CSync & Propagator: Support a direct download URL

Browse source 
This is for server file backends that support sending a
direct URL.
This commit is contained in:
Markus Goetz 2014-06-03 11:50:13 +02:00
parent 4d4ae9374b
commit 4d4eab8b1c
17 changed files with 168 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
csync/src/csync.c
View file

@ -442,6 +442,8 @@ static int _csync_treewalk_visitor(void *obj, void *data) {
trav.rename_path = cur->destpath;
trav.etag = cur->etag;
trav.file_id = cur->file_id;
trav.directDownloadUrl = cur->directDownloadUrl;
trav.directDownloadCookies = cur->directDownloadCookies;
trav.inode = cur->inode;
trav.error_status = cur->error_status;
@ -464,7 +466,7 @@ static int _csync_treewalk_visitor(void *obj, void *data) {
rc = (*visitor)(&trav, twctx->userdata);
cur->instruction = trav.instruction;
if (trav.etag != cur->etag) {
if (trav.etag != cur->etag) { // FIXME It would be nice to have this documented
SAFE_FREE(cur->etag);
cur->etag = c_strdup(trav.etag);
}
@ -912,6 +914,8 @@ int csync_abort_requested(CSYNC *ctx)
void csync_file_stat_free(csync_file_stat_t *st)
{
if (st) {
SAFE_FREE(st->directDownloadUrl);
SAFE_FREE(st->directDownloadCookies);
SAFE_FREE(st->etag);
SAFE_FREE(st->destpath);
SAFE_FREE(st);

2
csync/src/csync.h
View file

@ -189,6 +189,8 @@ struct csync_tree_walk_file_s {
const char *rename_path;
const char *etag;
const char *file_id;
char *directDownloadUrl;
char *directDownloadCookies;
struct {
int64_t size;
time_t modtime;

11
csync/src/csync_owncloud.c
View file

@ -515,6 +515,8 @@ static void results(void *userdata,
const char *resourcetype = NULL;
const char *md5sum = NULL;
const char *file_id = NULL;
const char *directDownloadUrl = NULL;
const char *directDownloadCookies = NULL;
const ne_status *status = NULL;
char *path = ne_path_unescape( uri->path );
@ -540,6 +542,8 @@ static void results(void *userdata,
resourcetype = ne_propset_value( set, &ls_props[2] );
md5sum = ne_propset_value( set, &ls_props[3] );
file_id = ne_propset_value( set, &ls_props[4] );
directDownloadUrl = ne_propset_value( set, &ls_props[5] );
directDownloadCookies = ne_propset_value( set, &ls_props[6] );
newres->type = resr_normal;
if( clength == NULL && resourcetype && strncmp( resourcetype, "<DAV:collection>", 16 ) == 0) {
@ -563,6 +567,13 @@ static void results(void *userdata,
csync_vio_set_file_id(newres->file_id, file_id);
if (directDownloadUrl) {
newres->directDownloadUrl = c_strdup(directDownloadUrl);
}
if (directDownloadCookies) {
newres->directDownloadCookies = c_strdup(directDownloadCookies);
}
/* prepend the new resource to the result list */
newres->next = fetchCtx->list;
fetchCtx->list = newres;

6
csync/src/csync_owncloud_private.h
View file

@ -125,6 +125,8 @@ static const ne_propname ls_props[] = {
{ "DAV:", "resourcetype" },
{ "DAV:", "getetag"},
{ "http://owncloud.org/ns", "id"},
{ "http://owncloud.org/ns", "directDownloadUrl"},
{ "http://owncloud.org/ns", "directDownloadCookies"},
{ NULL, NULL }
};
@ -140,6 +142,10 @@ typedef struct resource {
time_t modtime;
char* md5;
char file_id[FILE_ID_BUF_SIZE+1];
// Those two are optional from the server. We can use those URL to download the file directly
// without going through the ownCloud instance.
char *directDownloadUrl;
char *directDownloadCookies;
struct resource *next;
} resource;

12
csync/src/csync_owncloud_recursive_propfind.c
View file

@ -99,6 +99,8 @@ static void propfind_results_recursive(void *userdata,
{
struct resource *newres = 0;
const char *clength, *modtime, *file_id = NULL;
const char *directDownloadUrl = NULL;
const char *directDownloadCookies = NULL;
const char *resourcetype = NULL;
const char *md5sum = NULL;
const ne_status *status = NULL;
@ -109,6 +111,7 @@ static void propfind_results_recursive(void *userdata,
int depth = 0;
csync_owncloud_ctx_t *ctx = (csync_owncloud_ctx_t*) userdata;
(void) status;
if (!ctx->propfind_recursive_cache) {
@ -127,6 +130,8 @@ static void propfind_results_recursive(void *userdata,
resourcetype = ne_propset_value( set, &ls_props[2] );
md5sum = ne_propset_value( set, &ls_props[3] );
file_id = ne_propset_value( set, &ls_props[4] );
directDownloadUrl = ne_propset_value( set, &ls_props[5] );
directDownloadCookies = ne_propset_value( set, &ls_props[6] );
newres->type = resr_normal;
if( resourcetype && strncmp( resourcetype, "<DAV:collection>", 16 ) == 0) {
@ -157,6 +162,13 @@ static void propfind_results_recursive(void *userdata,
DEBUG_WEBDAV("propfind_results_recursive %s [%s] %s", newres->uri, newres->type == resr_collection ? "collection" : "file", newres->md5);
*/
if (directDownloadUrl) {
newres->directDownloadUrl = c_strdup(directDownloadUrl);
}
if (directDownloadCookies) {
newres->directDownloadCookies = c_strdup(directDownloadCookies);
}
/* Create new item in rb tree */
if (newres->type == resr_collection) {
DEBUG_WEBDAV("propfind_results_recursive %s is a folder", newres->uri);

19
csync/src/csync_owncloud_util.c
View file

@ -312,6 +312,15 @@ void resourceToFileStat(csync_vio_file_stat_t *lfs, struct resource *res )
}
csync_vio_file_stat_set_file_id(lfs, res->file_id);
if (res->directDownloadUrl) {
lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL;
lfs->directDownloadUrl = c_strdup(res->directDownloadUrl);
}
if (res->directDownloadCookies) {
lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES;
lfs->directDownloadCookies = c_strdup(res->directDownloadCookies);
}
}
/* WebDAV does not deliver permissions. Set a default here. */
@ -346,6 +355,12 @@ struct resource* resource_dup(struct resource* o) {
if( o->md5 ) {
r->md5 = c_strdup(o->md5);
}
if (o->directDownloadUrl) {
r->directDownloadUrl = c_strdup(o->directDownloadUrl);
}
if (o->directDownloadCookies) {
r->directDownloadCookies = c_strdup(o->directDownloadCookies);
}
r->next = o->next;
csync_vio_set_file_id(r->file_id, o->file_id);
@ -360,6 +375,8 @@ void resource_free(struct resource* o) {
SAFE_FREE(old->uri);
SAFE_FREE(old->name);
SAFE_FREE(old->md5);
SAFE_FREE(old->directDownloadUrl);
SAFE_FREE(old->directDownloadCookies);
SAFE_FREE(old);
}
}
@ -381,6 +398,8 @@ void free_fetchCtx( struct listdir_context *ctx )
SAFE_FREE(res->name);
SAFE_FREE(res->md5);
memset( res->file_id, 0, FILE_ID_BUF_SIZE+1 );
SAFE_FREE(res->directDownloadUrl);
SAFE_FREE(res->directDownloadCookies);
newres = res->next;
SAFE_FREE(res);

3
csync/src/csync_private.h
View file

@ -177,6 +177,9 @@ struct csync_file_stat_s {
char *destpath; /* for renames */
const char *etag;
char file_id[FILE_ID_BUF_SIZE+1]; /* the ownCloud file id is fixed width of 21 byte. */
char *directDownloadUrl;
char *directDownloadCookies;
CSYNC_STATUS error_status;
enum csync_instructions_e instruction; /* u32 */

9
csync/src/csync_update.c
View file

@ -369,6 +369,15 @@ out:
st->etag = c_strdup(fs->etag);
}
csync_vio_set_file_id(st->file_id, fs->file_id);
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL) {
SAFE_FREE(st->directDownloadUrl);
st->directDownloadUrl = c_strdup(fs->directDownloadUrl);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES) {
SAFE_FREE(st->directDownloadCookies);
st->directDownloadCookies = c_strdup(fs->directDownloadCookies);
}
fastout: /* target if the file information is read from database into st */
st->phash = h;

2
csync/src/vio/csync_vio.c
View file

@ -24,6 +24,7 @@
#include <errno.h>
#include <stdio.h>
#include <assert.h>
#include "csync_private.h"
#include "csync_util.h"
@ -107,6 +108,7 @@ int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) {
switch(ctx->replica) {
case REMOTE_REPLICA:
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "ERROR: Cannot call remote stat, not implemented");
assert(ctx->replica != REMOTE_REPLICA);
break;
case LOCAL_REPLICA:
rc = csync_vio_local_stat(uri, buf);

2
csync/src/vio/csync_vio_file_stat.c
View file

@ -36,6 +36,8 @@ void csync_vio_file_stat_destroy(csync_vio_file_stat_t *file_stat) {
if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_ETAG) {
SAFE_FREE(file_stat->etag);
}
SAFE_FREE(file_stat->directDownloadUrl);
SAFE_FREE(file_stat->directDownloadCookies);
SAFE_FREE(file_stat->name);
SAFE_FREE(file_stat);
}

6
csync/src/vio/csync_vio_file_stat.h
View file

@ -73,7 +73,9 @@ enum csync_vio_file_stat_fields_e {
// CSYNC_VIO_FILE_STAT_FIELDS_UID = 1 << 15,
// CSYNC_VIO_FILE_STAT_FIELDS_GID = 1 << 16,
CSYNC_VIO_FILE_STAT_FIELDS_ETAG = 1 << 17,
CSYNC_VIO_FILE_STAT_FIELDS_FILE_ID = 1 << 18
CSYNC_VIO_FILE_STAT_FIELDS_FILE_ID = 1 << 18,
CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL = 1 << 19,
CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES = 1 << 20
};
@ -81,6 +83,8 @@ struct csync_vio_file_stat_s {
char *name;
char *etag;
char file_id[FILE_ID_BUF_SIZE+1];
char *directDownloadUrl;
char *directDownloadCookies;
time_t atime;
time_t mtime;

9
src/creds/tokencredentials.cpp
View file

@ -83,7 +83,12 @@ protected:
QNetworkRequest req(request);
req.setRawHeader(QByteArray("Authorization"), QByteArray("Basic ") + credHash);
//qDebug() << "Request for " << req.url() << "with authorization" << QByteArray::fromBase64(credHash);
req.setRawHeader(QByteArray("Cookie"), _cred->_token.toLocal8Bit());
// Append token cookie
QList<QNetworkCookie> cookies = request.header(QNetworkRequest::CookieHeader).value<QList<QNetworkCookie> >();
cookies.append(QNetworkCookie::parseCookies(_cred->_token.toUtf8()));
req.setHeader(QNetworkRequest::CookieHeader, QVariant::fromValue(cookies));
return MirallAccessManager::createRequest(op, req, outgoingData);
}
private:
@ -112,7 +117,7 @@ void TokenCredentials::syncContextPreInit (CSYNC* ctx)
void TokenCredentials::syncContextPreStart (CSYNC* ctx)
{
csync_set_module_property(ctx, "session_key", _token.toLocal8Bit().data());
csync_set_module_property(ctx, "session_key", _token.toUtf8().data());
}
bool TokenCredentials::changed(AbstractCredentials* credentials) const

4
src/mirall/propagator_legacy.cpp
View file

@ -494,6 +494,10 @@ void PropagateDownloadFileLegacy::start()
_propagator->_journal->commit("download file start");
}
if (!_item._directDownloadUrl.isEmpty()) {
qDebug() << Q_FUNC_INFO << "Direct download URL" << _item._directDownloadUrl << "not supported with legacy propagator, will go via ownCloud server";
}
/* actually do the request */
int retry = 0;

86
src/mirall/propagator_qnam.cpp
View file

@ -358,13 +358,38 @@ void PropagateUploadFileQNAM::abort()
///////////////////////////////////////////////////////////////////////////////////////////////////
// DOES NOT take owncership of the device.
GETFileJob::GETFileJob(Account* account, const QString& path, QIODevice *device,
const QMap<QByteArray, QByteArray> &headers, QByteArray expectedEtagForResume,
QObject* parent)
: AbstractNetworkJob(account, path, parent),
_device(device), _headers(headers), _expectedEtagForResume(expectedEtagForResume),
_errorStatus(SyncFileItem::NoStatus)
{
}
GETFileJob::GETFileJob(Account* account, const QUrl& url, QIODevice *device,
const QMap<QByteArray, QByteArray> &headers,
QObject* parent)
: AbstractNetworkJob(account, url.toEncoded(), parent),
_device(device), _headers(headers),
_errorStatus(SyncFileItem::NoStatus), _directDownloadUrl(url)
{
}
void GETFileJob::start() {
QNetworkRequest req;
for(QMap<QByteArray, QByteArray>::const_iterator it = _headers.begin(); it != _headers.end(); ++it) {
req.setRawHeader(it.key(), it.value());
}
setReply(davRequest("GET", path(), req));
if (_directDownloadUrl.isEmpty()) {
setReply(davRequest("GET", path(), req));
} else {
// Use direct URL
setReply(davRequest("GET", _directDownloadUrl, req));
}
setupConnections(reply());
if( reply()->error() != QNetworkReply::NoError ) {
@ -386,17 +411,21 @@ void GETFileJob::slotMetaDataChanged()
return;
}
QByteArray etag = parseEtag(reply()->rawHeader("Etag"));
if (etag.isEmpty()) {
_etag = parseEtag(reply()->rawHeader("Etag"));
if (!_directDownloadUrl.isEmpty() && !_etag.isEmpty()) {
qDebug() << Q_FUNC_INFO << "Direct download used, ignoring server ETag" << _etag;
_etag = QByteArray(); // reset received ETag
} else if (!_directDownloadUrl.isEmpty()) {
// All fine, ETag empty and directDownloadUrl used
} else if (_etag.isEmpty()) {
qDebug() << Q_FUNC_INFO << "No E-Tag reply by server, considering it invalid";
_errorString = tr("No E-Tag received from server, check Proxy/Gateway");
_errorStatus = SyncFileItem::NormalError;
reply()->abort();
return;
} else if (!_expectedEtagForResume.isEmpty() && _expectedEtagForResume != etag) {
} else if (!_expectedEtagForResume.isEmpty() && _expectedEtagForResume != _etag) {
qDebug() << Q_FUNC_INFO << "We received a different E-Tag for resuming!"
<< _expectedEtagForResume << "vs" << etag;
<< _expectedEtagForResume << "vs" << _etag;
_errorString = tr("We received a different E-Tag for resuming. Retrying next time.");
_errorStatus = SyncFileItem::NormalError;
reply()->abort();
@ -431,6 +460,13 @@ void GETFileJob::slotReadyRead()
resetTimeout();
}
void GETFileJob::slotTimeout()
{
_errorString = tr("Connection Timeout");
_errorStatus = SyncFileItem::FatalError;
reply()->abort();
}
void PropagateDownloadFileQNAM::start()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
@ -438,7 +474,7 @@ void PropagateDownloadFileQNAM::start()
qDebug() << Q_FUNC_INFO << _item._file << _propagator->_activeJobs;
// do a case clash check.
// do a klaas' case clash check.
if( _propagator->localFileNameClash(_item._file) ) {
done( SyncFileItem::NormalError, tr("File %1 can not be downloaded because of a local file name clash!")
.arg(QDir::toNativeSeparators(_item._file)) );
@ -490,8 +526,6 @@ void PropagateDownloadFileQNAM::start()
QMap<QByteArray, QByteArray> headers;
/* Allow compressed content by setting the header */
//headers["Accept-Encoding"] = "gzip";
if (_tmpFile.size() > 0) {
quint64 done = _tmpFile.size();
@ -506,9 +540,22 @@ void PropagateDownloadFileQNAM::start()
_startSize = done;
}
_job = new GETFileJob(AccountManager::instance()->account(),
_propagator->_remoteFolder + _item._file,
&_tmpFile, headers, expectedEtagForResume);
if (_item._directDownloadUrl.isEmpty()) {
// Normal job, download from oC instance
_job = new GETFileJob(AccountManager::instance()->account(),
_propagator->_remoteFolder + _item._file,
&_tmpFile, headers, expectedEtagForResume);
} else {
// We were provided a direct URL, use that one
if (!_item._directDownloadCookies.isEmpty()) {
headers["Cookie"] = _item._directDownloadCookies.toUtf8();
}
QUrl url = QUrl::fromUserInput(_item._directDownloadUrl);
_job = new GETFileJob(AccountManager::instance()->account(),
url,
&_tmpFile, headers);
qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item._file << _item._directDownloadUrl;
}
_job->setTimeout(_propagator->httpTimeout() * 1000);
connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotGetFinished()));
connect(_job, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotDownloadProgress(qint64,qint64)));
@ -546,7 +593,11 @@ void PropagateDownloadFileQNAM::slotGetFinished()
return;
}
_item._etag = parseEtag(job->reply()->rawHeader("Etag"));
if (!job->etag().isEmpty()) {
// The etag will be empty if we used a direct download URL.
// (If it was really empty by the server, the GETFileJob will have errored
_item._etag = parseEtag(job->etag());
}
_item._requestDuration = job->duration();
_item._responseTimeStamp = job->responseTimestamp();
@ -628,13 +679,4 @@ void PropagateDownloadFileQNAM::abort()
_job->reply()->abort();
}
void GETFileJob::slotTimeout()
{
_errorString = tr("Connection Timeout");
_errorStatus = SyncFileItem::FatalError;
reply()->abort();
}
}

13
src/mirall/propagator_qnam.h
View file

@ -110,15 +110,18 @@ class GETFileJob : public AbstractNetworkJob {
QString _errorString;
QByteArray _expectedEtagForResume;
SyncFileItem::Status _errorStatus;
QUrl _directDownloadUrl;
QByteArray _etag;
public:
// DOES NOT take owncership of the device.
explicit GETFileJob(Account* account, const QString& path, QIODevice *device,
const QMap<QByteArray, QByteArray> &headers, QByteArray expectedEtagForResume,
QObject* parent = 0)
: AbstractNetworkJob(account, path, parent),
_device(device), _headers(headers), _expectedEtagForResume(expectedEtagForResume),
_errorStatus(SyncFileItem::NoStatus) {}
QObject* parent = 0);
// For directDownloadUrl:
explicit GETFileJob(Account* account, const QUrl& url, QIODevice *device,
const QMap<QByteArray, QByteArray> &headers,
QObject* parent = 0);
virtual void start();
virtual bool finished() {
@ -134,6 +137,8 @@ public:
virtual void slotTimeout();
QByteArray &etag() { return _etag; }
signals:
void finishedSignal();

6
src/mirall/syncengine.cpp
View file

@ -263,6 +263,12 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
item._instruction = file->instruction;
item._direction = SyncFileItem::None;
item._fileId = file->file_id;
if (file->directDownloadUrl) {
item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl );
}
if (file->directDownloadCookies) {
item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies );
}
// record the seen files to be able to clean the journal later
_seenFiles[item._file] = QString();

2
src/mirall/syncfileitem.h
View file

@ -86,6 +86,8 @@ public:
quint64 _inode;
bool _should_update_etag;
QByteArray _fileId;
QString _directDownloadUrl;
QString _directDownloadCookies;
bool _blacklistedInDb;
// Variables usefull to report to the user