diff --git a/modules/csync_owncloud.c b/modules/csync_owncloud.c index dec65b019..96e257499 100644 --- a/modules/csync_owncloud.c +++ b/modules/csync_owncloud.c @@ -43,6 +43,8 @@ #include #include +#include "c_rbtree.h" + #include "c_lib.h" #include "csync.h" #include "csync_misc.h" @@ -191,11 +193,18 @@ long long chunked_total_size = 0; long long chunked_done = 0; struct listdir_context *propfind_cache = 0; + +bool is_first_propfind = true; +static void clear_propfind_recursive_cache(); + csync_vio_file_stat_t _stat_cache; /* id cache, cache the ETag: header of a GET request */ struct { char *uri; char *id; } _id_cache = { NULL, NULL }; static void clean_caches() { + clear_propfind_recursive_cache(); + is_first_propfind = true; + free_fetchCtx(propfind_cache); propfind_cache = NULL; @@ -377,6 +386,10 @@ static char *_cleanPath( const char* uri ) { SAFE_FREE( path ); return re; } +/* ***************************************************************************** */ +#include "csync_owncloud_recursive_propfind.c" +/* ***************************************************************************** */ + /* * helper method to build up a user text for SSL problems, called from the @@ -992,6 +1005,20 @@ static struct listdir_context *fetch_resource_list(const char *uri, int depth) } } + if (propfind_recursive_cache) { + fetchCtx = get_listdir_context_from_cache(curi); + if (fetchCtx) { + return fetchCtx; + } else { + /* Not found in the recursive cache, fetch some */ + return fetch_resource_list_recursive(uri, curi); + } + } else if (!is_first_propfind) { + /* 2nd propfind */ + return fetch_resource_list_recursive(uri, curi); + } + is_first_propfind = false; + fetchCtx = c_malloc( sizeof( struct listdir_context )); if (!fetchCtx) { errno = ENOMEM; diff --git a/modules/csync_owncloud_recursive_propfind.c b/modules/csync_owncloud_recursive_propfind.c new file mode 100644 index 000000000..617e65784 --- /dev/null +++ b/modules/csync_owncloud_recursive_propfind.c @@ -0,0 +1,272 @@ +static time_t oc_httpdate_parse( const char *date ); + +// Our cache, key is a char* +c_rbtree_t *propfind_recursive_cache = NULL; +// Values are propfind_recursive_element: +struct propfind_recursive_element { + struct resource *self; + struct resource *children; +}; +typedef struct propfind_recursive_element propfind_recursive_element_t; + +static struct resource* resource_dup(struct resource* o) { + struct resource *r = c_malloc (sizeof( struct resource )); + r->uri = c_strdup(o->uri); + r->name = c_strdup(o->name); + r->type = o->type; + r->size = o->size; + r->modtime = o->modtime; + r->md5 = c_strdup(o->md5); + r->next = o->next; + return r; +} +static void resource_free(struct resource* o) { + struct resource* old = NULL; + while (o) + { + old = o; + o = o->next; + SAFE_FREE(old->uri); + SAFE_FREE(old->name); + SAFE_FREE(old->md5); + SAFE_FREE(old); + } +} + +static void _tree_destructor(void *data) { + propfind_recursive_element_t *element = data; + resource_free(element->self); + resource_free(element->children); + SAFE_FREE(element); +} +static void clear_propfind_recursive_cache() +{ + c_rbtree_destroy(propfind_recursive_cache, _tree_destructor); + propfind_recursive_cache = NULL; +} + +static struct listdir_context *get_listdir_context_from_cache(const char *curi) +{ + if (!propfind_recursive_cache) { + DEBUG_WEBDAV("get_listdir_context_from_cache No cache"); + return NULL; + } + + propfind_recursive_element_t *element = c_rbtree_node_data(c_rbtree_find(propfind_recursive_cache, curi)); + if (!element) { + DEBUG_WEBDAV("get_listdir_context_from_cache No element %s in cache found", curi); + return NULL; + } + if (!element->children) { + DEBUG_WEBDAV("get_listdir_context_from_cache Element %s in cache found, but no children, assuming that recursive propfind didn't work", curi); + return NULL; + } + + // Out of the element, create a listdir_context.. if we could be sure that it is immutable, we could ref instead.. need to investigate + struct listdir_context *fetchCtx = c_malloc( sizeof( struct listdir_context )); + fetchCtx->list = NULL; + fetchCtx->target = c_strdup(curi); + fetchCtx->currResource = NULL; + fetchCtx->ref = 1; + + struct resource *iterator = element->children; + struct resource *r = NULL; + while (iterator) { + r = resource_dup(iterator); + r->next = fetchCtx->list; + fetchCtx->list = r; + iterator = iterator->next; + fetchCtx->result_count++; + /* DEBUG_WEBDAV("get_listdir_context_from_cache Returning cache for %s element %s", fetchCtx->target, fetchCtx->list->uri); */ + } + r = resource_dup(element->self); + r->next = fetchCtx->list; + fetchCtx->result_count++; + fetchCtx->list = r; + fetchCtx->currResource = fetchCtx->list; + DEBUG_WEBDAV("get_listdir_context_from_cache Returning cache for %s (%d elements)", fetchCtx->target, fetchCtx->result_count); + return fetchCtx; +} + +static int _key_cmp(const void *key, const void *b) { + const char *elementAUri = (char*)key; + const propfind_recursive_element_t *elementB = b; + return ne_path_compare(elementAUri, elementB->self->uri); +} +static int _data_cmp(const void *a, const void *b) { + const propfind_recursive_element_t *elementA = a; + const propfind_recursive_element_t *elementB = b; + return ne_path_compare(elementA->self->uri, elementB->self->uri); +} +static void results_recursive(void *userdata, + const ne_uri *uri, + const ne_prop_result_set *set) +{ + struct resource *newres = 0; + const char *clength, *modtime = NULL; + const char *resourcetype = NULL; + const char *md5sum = NULL; + const ne_status *status = NULL; + char *path = ne_path_unescape( uri->path ); + + (void) status; + + if (!propfind_recursive_cache) { + c_rbtree_create(&propfind_recursive_cache, _key_cmp, _data_cmp); + } + + /* Fill the resource structure with the data about the file */ + newres = c_malloc(sizeof(struct resource)); + newres->uri = path; /* no need to strdup because ne_path_unescape already allocates */ + newres->name = c_basename( path ); + + modtime = ne_propset_value( set, &ls_props[0] ); + clength = ne_propset_value( set, &ls_props[1] ); + resourcetype = ne_propset_value( set, &ls_props[2] ); + md5sum = ne_propset_value( set, &ls_props[3] ); + + newres->type = resr_normal; + if( resourcetype && strncmp( resourcetype, "", 16 ) == 0) { + newres->type = resr_collection; + } else { + DEBUG_WEBDAV("results_recursive %s [%d]", newres->uri, newres->type); + } + + if (modtime) { + newres->modtime = oc_httpdate_parse(modtime); + } + + /* DEBUG_WEBDAV("Parsing Modtime: %s -> %llu", modtime, (unsigned long long) newres->modtime ); */ + + if (clength) { + char *p; + + newres->size = DAV_STRTOL(clength, &p, 10); + if (*p) { + newres->size = 0; + } + /* DEBUG_WEBDAV("Parsed File size for %s from %s: %lld", newres->name, clength, (long long)newres->size ); */ + } + + if( md5sum ) { + int len = strlen(md5sum)-2; + if( len > 0 ) { + /* Skip the " around the string coming back from the ne_propset_value call */ + newres->md5 = c_malloc(len+1); + strncpy( newres->md5, md5sum+1, len ); + newres->md5[len] = '\0'; + } + } + + DEBUG_WEBDAV("results_recursive %s [%s] >%s<", newres->uri, newres->type == resr_collection ? "collection" : "file", resourcetype); + + /* Create new item in rb tree */ + if (newres->type == resr_collection) { + DEBUG_WEBDAV("results_recursiveIt is a collection %s", newres->uri); + // Check if in rb tree + propfind_recursive_element_t *element = c_rbtree_node_data(c_rbtree_find(propfind_recursive_cache,uri->path)); + // If not, create a new item and insert it + if (!element) { + element = c_malloc(sizeof(propfind_recursive_element_t)); + element->self = resource_dup(newres); + element->children = NULL; + c_rbtree_insert(propfind_recursive_cache, element); + /* DEBUG_WEBDAV("results_recursive Added collection %s", newres->uri); */ + } + } + + /* Check for parent in tree. If exists: Insert it into the children elements there */ + char *parentPath = ne_path_parent(uri->path); + if (parentPath) { + free(parentPath); + propfind_recursive_element_t *element = c_rbtree_node_data(c_rbtree_find(propfind_recursive_cache,parentPath)); + if (element) { + newres->next = element->children; + element->children = newres; + /* DEBUG_WEBDAV("results_recursive Added child %s to collection %s", newres->uri, element->self->uri); */ + } else { + /* DEBUG_WEBDAV("results_recursive No parent %s found for child %s", parentPath, newres->uri); */ + resource_free(newres); + newres = NULL; + } + } + +} + + +/* + * fetches a resource list from the WebDAV server. This is equivalent to list dir. + */ +static struct listdir_context *fetch_resource_list_recursive(const char *uri, const char *curi) +{ + int ret = 0; + ne_propfind_handler *hdl = NULL; + ne_request *request = NULL; + const char *content_type = NULL; + const ne_status *req_status = NULL; + int depth = NE_DEPTH_INFINITE; + + DEBUG_WEBDAV("Starting recursive propfind %s %s", uri, curi); + + /* do a propfind request and parse the results in the results function, set as callback */ + hdl = ne_propfind_create(dav_session.ctx, curi, depth); + + if(hdl) { + ret = ne_propfind_named(hdl, ls_props, results_recursive, NULL); + request = ne_propfind_get_request( hdl ); + req_status = ne_get_status( request ); + } + + if( ret == NE_OK ) { + /* Check the request status. */ + if( req_status && req_status->klass != 2 ) { + set_errno_from_http_errcode(req_status->code); + DEBUG_WEBDAV("ERROR: Request failed: status %d (%s)", req_status->code, + req_status->reason_phrase); + ret = NE_CONNECT; + set_error_message(req_status->reason_phrase); + if (_progresscb) { + _progresscb(uri, CSYNC_NOTIFY_ERROR, req_status->code, (long long)(req_status->reason_phrase) ,dav_session.userdata); + } + } + DEBUG_WEBDAV("Recursive propfind result code %d.", req_status->code); + } else { + if( ret == NE_ERROR && req_status->code == 404) { + errno = ENOENT; + } else { + set_errno_from_neon_errcode(ret); + } + } + + if( ret == NE_OK ) { + /* Check the content type. If the server has a problem, ie. database is gone or such, + * the content type is not xml but a html error message. Stop on processing if it's + * not XML. + * FIXME: Generate user error message from the reply content. + */ + content_type = ne_get_response_header( request, "Content-Type" ); + if( !(content_type && c_streq(content_type, "application/xml; charset=utf-8") ) ) { + DEBUG_WEBDAV("ERROR: Content type of propfind request not XML: %s.", + content_type ? content_type: ""); + errno = ERRNO_WRONG_CONTENT; + set_error_message("Server error: PROPFIND reply is not XML formatted!"); + ret = NE_CONNECT; + } + } + + if( ret != NE_OK ) { + const char *err = NULL; + + err = ne_get_error( dav_session.ctx ); + DEBUG_WEBDAV("WRN: propfind named failed with %d, request error: %s", ret, err ? err : ""); + } + + if( hdl ) + ne_propfind_destroy(hdl); + + if( ret != NE_OK ) { + return NULL; + } + + return get_listdir_context_from_cache(curi); +}