Add a progress callback for overall and individual file up and download

progress.

modules/csync_owncloud.c

@@ -59,7 +59,7 @@ int _connected = 0;                   /* flag to indicate if a connection exists
                                      the dav_session is valid */
 
 csync_auth_callback _authcb;
-csync_progress_callback _progresscb;
+csync_file_progress_callback _file_progress_cb;
 long long chunked_total_size = 0;
 long long chunked_done = 0;
 
@@ -397,9 +397,9 @@ static void ne_notify_status_cb (void *userdata, ne_session_status status,
 {
     struct transfer_context *tc = (struct transfer_context*) userdata;
 
-    if (_progresscb && (status == ne_status_sending || status == ne_status_recving)) {
+    if (_file_progress_cb && (status == ne_status_sending || status == ne_status_recving)) {
         if (info->sr.total > 0)
-            _progresscb(tc->url, CSYNC_NOTIFY_PROGRESS,
+            _file_progress_cb(tc->url, CSYNC_NOTIFY_PROGRESS,
                         chunked_done + info->sr.progress,
                         chunked_total_size ? chunked_total_size : info->sr.total,
                         csync_get_userdata(dav_session.csync_ctx));
@@ -690,8 +690,8 @@ static struct listdir_context *fetch_resource_list(const char *uri, int depth)
                          req_status->reason_phrase);
             ret = NE_CONNECT;
             set_error_message(req_status->reason_phrase);
-            if (_progresscb) {
+            if (_file_progress_cb) {
-                _progresscb(uri, CSYNC_NOTIFY_ERROR,  req_status->code, (long long)(req_status->reason_phrase) ,
+                _file_progress_cb(uri, CSYNC_NOTIFY_ERROR,  req_status->code, (long long)(req_status->reason_phrase) ,
                             csync_get_userdata(dav_session.csync_ctx));
             }
         }
@@ -1217,9 +1217,9 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
             trans->transfer_id = dav_session.chunk_info->transfer_id;
           }
 
-          if (state == HBF_SUCCESS && _progresscb) {
+          if (state == HBF_SUCCESS && _file_progress_cb) {
             ne_set_notifier(dav_session.ctx, ne_notify_status_cb, write_ctx);
-            _progresscb(write_ctx->url, CSYNC_NOTIFY_START_UPLOAD, 0 , 0, csync_get_userdata(dav_session.csync_ctx));
+            _file_progress_cb(write_ctx->url, CSYNC_NOTIFY_START_UPLOAD, 0 , 0, csync_get_userdata(dav_session.csync_ctx));
           }
 
           /* Register the abort callback */
@@ -1262,9 +1262,9 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
         hbf_free_transfer(trans);
       } while( !finished );
 
-      if (_progresscb) {
+      if (_file_progress_cb) {
         ne_set_notifier(dav_session.ctx, 0, 0);
-        _progresscb(write_ctx->url, rc != 0 ? CSYNC_NOTIFY_ERROR :
+        _file_progress_cb(write_ctx->url, rc != 0 ? CSYNC_NOTIFY_ERROR :
                                               CSYNC_NOTIFY_FINISHED_UPLOAD, error_code,
                     (long long)(error_string), csync_get_userdata(dav_session.csync_ctx));
       }
@@ -1274,9 +1274,9 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
       int retry = 0;
       DEBUG_WEBDAV("  -- GET on %s", write_ctx->url);
       write_ctx->fd = fd;
-      if (_progresscb) {
+      if (_file_progress_cb) {
         ne_set_notifier(dav_session.ctx, ne_notify_status_cb, write_ctx);
-        _progresscb(write_ctx->url, CSYNC_NOTIFY_START_DOWNLOAD, 0 , 0, csync_get_userdata(dav_session.csync_ctx));
+        _file_progress_cb(write_ctx->url, CSYNC_NOTIFY_START_DOWNLOAD, 0 , 0, csync_get_userdata(dav_session.csync_ctx));
       }
 
       do {
@@ -1353,9 +1353,9 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
         }
         break;
       } while (1);
-      if (_progresscb) {
+      if (_file_progress_cb) {
           ne_set_notifier(dav_session.ctx, 0, 0);
-          _progresscb(write_ctx->url, (rc != NE_OK) ? CSYNC_NOTIFY_ERROR :
+          _file_progress_cb(write_ctx->url, (rc != NE_OK) ? CSYNC_NOTIFY_ERROR :
                       CSYNC_NOTIFY_FINISHED_DOWNLOAD, error_code ,
                       (long long)(error_string), csync_get_userdata(dav_session.csync_ctx));
       }
@@ -1520,8 +1520,8 @@ static int owncloud_mkdir(const char *uri, mode_t mode) {
          * To keep csync vio_mkdirs working errno EEXIST has to be returned. */
         if (errno == EPERM && http_result_code_from_session() == 405) {
             errno = EEXIST;
-        } else if (rc != NE_OK && _progresscb) {
+        } else if (rc != NE_OK && _file_progress_cb) {
-            _progresscb(uri, CSYNC_NOTIFY_ERROR,  http_result_code_from_session(),
+            _file_progress_cb(uri, CSYNC_NOTIFY_ERROR,  http_result_code_from_session(),
                         (long long)(dav_session.error_string) , csync_get_userdata(dav_session.csync_ctx));
         }
     }
@@ -1580,8 +1580,8 @@ static int owncloud_rename(const char *olduri, const char *newuri) {
             errno = ENOENT;
         } else {
             set_errno_from_neon_errcode(rc);
-            if (rc != NE_OK && _progresscb) {
+            if (rc != NE_OK && _file_progress_cb) {
-                _progresscb(olduri, CSYNC_NOTIFY_ERROR,  http_result_code_from_session(),
+                _file_progress_cb(olduri, CSYNC_NOTIFY_ERROR,  http_result_code_from_session(),
                             (long long)(dav_session.error_string) ,csync_get_userdata(dav_session.csync_ctx));
             }
         }
@@ -1725,7 +1725,7 @@ static int owncloud_set_property(const char *key, void *data) {
         return 0;
     }
     if (c_streq(key, "progress_callback")) {
-        _progresscb = *(csync_progress_callback*)(data);
+        _file_progress_cb = *(csync_file_progress_callback*)(data);
         return 0;
     }
     if (c_streq(key, "read_timeout") || c_streq(key, "timeout")) {
@@ -1735,7 +1735,7 @@ static int owncloud_set_property(const char *key, void *data) {
     if( c_streq(key, "csync_context")) {
         dav_session.csync_ctx = data;
         if( data ) {
-          _progresscb = csync_get_progress_callback(dav_session.csync_ctx);
+          _file_progress_cb = csync_get_file_progress_callback(dav_session.csync_ctx);
         }
         return 0;
     }

modules/csync_owncloud_recursive_propfind.c

@@ -224,7 +224,8 @@ static void results_recursive(void *userdata,
 /*
  * fetches a resource list from the WebDAV server. This is equivalent to list dir.
  */
-extern csync_progress_callback _progresscb;
+extern csync_file_progress_callback _file_progress_cb;
+
 struct listdir_context *fetch_resource_list_recursive(const char *uri, const char *curi)
 {
     int ret = 0;
@@ -253,8 +254,8 @@ struct listdir_context *fetch_resource_list_recursive(const char *uri, const cha
                          req_status->reason_phrase);
             ret = NE_CONNECT;
             set_error_message(req_status->reason_phrase);
-            if (_progresscb) {
+            if (_file_progress_cb) {
-                _progresscb(uri, CSYNC_NOTIFY_ERROR,  req_status->code, (long long)(req_status->reason_phrase),
+                _file_progress_cb(uri, CSYNC_NOTIFY_ERROR,  req_status->code, (long long)(req_status->reason_phrase),
                             csync_get_userdata(dav_session.csync_ctx));
             }
         }

src/csync.c

@@ -333,12 +333,30 @@ retry_vio_init:
     }
   }
 
-  if (ctx->callbacks.progresscb)
-    csync_vio_set_property(ctx, "progress_callback", &ctx->callbacks.progresscb);
-
   if (ctx->options.timeout)
     csync_vio_set_property(ctx, "timeout", &ctx->options.timeout);
 
+  /* Install progress callbacks in the module. */
+  if (ctx->callbacks.file_progress_cb != NULL) {
+      int prc;
+      prc = csync_vio_set_property(ctx, "file_progress_callback", &ctx->callbacks.file_progress_cb);
+      if (prc == -1) {
+          /* The module does not support the callbacks */
+          CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Could not install file progress callback!");
+          ctx->callbacks.file_progress_cb = NULL;
+      }
+  }
+
+  if (ctx->callbacks.overall_progress_cb != NULL) {
+      int prc;
+      prc = csync_vio_set_property(ctx, "overall_progress_callback", &ctx->callbacks.overall_progress_cb);
+      if (prc == -1) {
+          /* The module does not support the callbacks */
+          CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Could not install overall progress callback!");
+          ctx->callbacks.overall_progress_cb = NULL;
+      }
+  }
+
   if (c_rbtree_create(&ctx->local.tree, _key_cmp, _data_cmp) < 0) {
     ctx->error_code = CSYNC_ERR_TREE;
     rc = -1;
@@ -765,10 +783,10 @@ int csync_commit(CSYNC *ctx) {
 
   csync_vio_commit(ctx);
 
-  while (ctx->progress) {
+  while (ctx->progress_info) {
-    csync_progressinfo_t *next = ctx->progress->next;
+    csync_progressinfo_t *next = ctx->progress_info->next;
-    csync_statedb_free_progressinfo(ctx->progress);
+    csync_statedb_free_progressinfo(ctx->progress_info);
-    ctx->progress = next;
+    ctx->progress_info = next;
   }
 
   /* destroy the rbtrees */
@@ -826,6 +844,12 @@ int csync_commit(CSYNC *ctx) {
     goto out;
   }
 
+  /* reset the progress */
+  ctx->progress.file_count = 0;
+  ctx->progress.current_file_no = 0;
+  ctx->progress.byte_sum = 0;
+  ctx->progress.byte_current = 0;
+
   ctx->status = CSYNC_STATUS_INIT;
   ctx->error_code = CSYNC_ERR_NONE;
   SAFE_FREE(ctx->error_string);
@@ -870,10 +894,10 @@ int csync_destroy(CSYNC *ctx) {
     csync_lock_remove(ctx, lock);
   }
 
-  while (ctx->progress) {
+  while (ctx->progress_info) {
-    csync_progressinfo_t *next = ctx->progress->next;
+    csync_progressinfo_t *next = ctx->progress_info->next;
-    csync_statedb_free_progressinfo(ctx->progress);
+    csync_statedb_free_progressinfo(ctx->progress_info);
-    ctx->progress = next;
+    ctx->progress_info = next;
   }
 
   /* destroy the rbtrees */
@@ -1188,32 +1212,6 @@ int csync_set_iconv_codec(const char *from)
 }
 #endif
 
-csync_progress_callback csync_get_progress_callback(CSYNC *ctx)
-{
-  if ( ctx==NULL ) {
-    return NULL;
-  }
-  return ctx->callbacks.progresscb;
-}
-
-
-int csync_set_progress_callback(CSYNC* ctx, csync_progress_callback cb)
-{
-  if (ctx == NULL) {
-    return -1;
-  }
-  if (cb == NULL ) {
-    ctx->error_code = CSYNC_ERR_PARAM;
-    return -1;
-  }
-
-  ctx->error_code = CSYNC_ERR_NONE;
-  ctx->callbacks.progresscb = cb;
-
-  return 0;
-
-}
-
 void csync_request_abort(CSYNC *ctx)
 {
   if (ctx != NULL) {
@@ -1247,4 +1245,44 @@ void csync_file_stat_free(csync_file_stat_t *st)
   }
 }
 
+int csync_set_file_progress_callback(CSYNC* ctx, csync_file_progress_callback cb)
+{
+  if (ctx == NULL) {
+    return -1;
+  }
+  if (cb == NULL ) {
+    ctx->error_code = CSYNC_ERR_PARAM;
+    return -1;
+  }
+
+  ctx->callbacks.file_progress_cb = cb;
+
+  return 0;
+
+}
+
+csync_file_progress_callback csync_get_file_progress_callback(CSYNC *ctx)
+{
+  if (ctx == NULL) {
+    return NULL;
+  }
+
+  return ctx->callbacks.file_progress_cb;
+}
+
+int csync_set_overall_progress_callback(CSYNC* ctx, csync_overall_progress_callback cb)
+{
+  if (ctx == NULL) {
+    return -1;
+  }
+  if (cb == NULL ) {
+    ctx->error_code = CSYNC_ERR_PARAM;
+    return -1;
+  }
+
+  ctx->callbacks.overall_progress_cb = cb;
+
+  return 0;
+
+}
 /* vim: set ts=8 sw=2 et cindent: */

src/csync.h

@@ -518,27 +518,54 @@ int csync_set_module_property(CSYNC *ctx, const char *key, void *value);
 enum csync_notify_type_e { CSYNC_NOTIFY_START_DOWNLOAD, CSYNC_NOTIFY_START_UPLOAD,
                            CSYNC_NOTIFY_PROGRESS, CSYNC_NOTIFY_FINISHED_DOWNLOAD,
                            CSYNC_NOTIFY_FINISHED_UPLOAD, CSYNC_NOTIFY_ERROR };
-
+/**
-typedef void (*csync_progress_callback) (const char *remote_url, enum csync_notify_type_e kind,
+ * @brief Callback definition for individual file progress callback.
-                                        long long o1, long long o2, void *userdata);
+ *
+ * @param remote_url    The currently handled file.
+ *
+ * @param kind          The type of progress.
+ *
+ * @param o1            The current transmitted bytes.
+ *
+ * @param o2            The size of the file.
+ */
+typedef void (*csync_file_progress_callback) (const char *remote_url, enum csync_notify_type_e kind,
+                                              long long o1, long long o2, void *userdata);
 
 /**
- * @brief Get the progress callback from the context.
+ * @brief Set a progress callback for individual files.
+ *
+ * This callback reports about up- or download progress of a individual file.
+ */
+
+int csync_set_file_progress_callback(CSYNC* ctx, csync_file_progress_callback cb);
+
+csync_file_progress_callback csync_get_file_progress_callback(CSYNC *ctx);
+
+/**
+ * @brief Callback definition for overall progress callback.
+ *
+ * @param file_no       The current number of up- or downloaded files.
+ *
+ * @param file_cnt      The overall number of files to transmit.
+ *
+ * @param o1            The current transmitted bytes.
+ *
+ * @param o2            The overall sum of bytes to transmit.
+ */
+typedef void (*csync_overall_progress_callback) (const char *file_name, int file_no,
+                                                 int file_cnt, long long o1, long long o2);
+
+/**
+ * @brief Set a progress callback for the overall files.
+ *
+ * This callback reports about overall up- or download progress.
  *
  * @param ctx           The csync context.
  *
  * @param cb            The callback
  */
-csync_progress_callback csync_get_progress_callback(CSYNC *ctx);
+int csync_set_overall_progress_callback (CSYNC* ctx, csync_overall_progress_callback cb);
-
-/**
- * @brief Set a progress callback
- *
- * @param ctx           The csync context.
- *
- * @param cb            The callback
- */
-int csync_set_progress_callback(CSYNC *ctx, csync_progress_callback cb);
 
 /**
  * @brief Aborts the current sync run as soon as possible. Can be called from another thread.

src/csync_private.h

@@ -89,7 +89,9 @@ struct csync_s {
   struct {
       csync_auth_callback auth_function;
       csync_log_callback log_function;
-      csync_progress_callback progresscb;
+      csync_file_progress_callback progresscb;
+      csync_overall_progress_callback overall_progress_cb;
+      csync_file_progress_callback file_progress_cb;
       void *userdata;
   } callbacks;
   c_strlist_t *excludes;
@@ -144,7 +146,14 @@ struct csync_s {
     uid_t euid;
   } pwd;
 
-  struct csync_progressinfo_s *progress;
+  struct {
+    int file_count;
+    int current_file_no;
+    long long byte_sum;
+    long long byte_current;
+  } progress;
+
+  struct csync_progressinfo_s *progress_info;
 
   /* replica we are currently walking */
   enum csync_replica_e current;

src/csync_propagate.c

@@ -119,8 +119,8 @@ static void _csync_record_error(CSYNC *ctx, csync_file_stat_t *st, csync_progres
     pi->error = 1;
   }
   pi->error_string = st->error_string ? c_strdup(st->error_string) : NULL;
-  pi->next = ctx->progress;
+  pi->next = ctx->progress_info;
-  ctx->progress = pi;
+  ctx->progress_info = pi;
 }
 
 static bool _push_to_tmp_first(CSYNC *ctx)
@@ -656,6 +656,13 @@ start_fd_based:
   /* set instruction for the statedb merger */
   st->instruction = CSYNC_INSTRUCTION_UPDATED;
 
+  /* Notify the overall progress */
+  if (ctx->callbacks.overall_progress_cb) {
+      ctx->progress.byte_current += st->size;
+      ctx->callbacks.overall_progress_cb(duri, ++ctx->progress.current_file_no, ctx->progress.file_count,
+                                         ctx->progress.byte_current, ctx->progress.byte_sum);
+  }
+
   CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "PUSHED  file: %s", duri);
 
   rc = 0;
@@ -1456,6 +1463,48 @@ static int _csync_propagation_cleanup(CSYNC *ctx) {
   return 0;
 }
 
+static int _csync_propagation_file_count_visitor(void *obj, void *data) {
+  csync_file_stat_t *st = NULL;
+  CSYNC *ctx = NULL;
+
+  st = (csync_file_stat_t *) obj;
+  ctx = (CSYNC *) data;
+
+  if (st == NULL) {
+      return -1;
+  }
+  if (ctx == NULL) {
+      return -1;
+  }
+
+  switch(st->type) {
+    case CSYNC_FTW_TYPE_SLINK:
+      break;
+    case CSYNC_FTW_TYPE_FILE:
+      switch (st->instruction) {
+        case CSYNC_INSTRUCTION_NEW:
+        case CSYNC_INSTRUCTION_SYNC:
+        case CSYNC_INSTRUCTION_CONFLICT:
+          ctx->progress.file_count++;
+          ctx->progress.byte_sum += st->size;
+          break;
+        default:
+          break;
+      }
+      break;
+    case CSYNC_FTW_TYPE_DIR:
+      /*
+       * No counting of directories.
+       */
+      break;
+    default:
+      break;
+  }
+
+  return 0;
+}
+
+
 static int _csync_propagation_file_visitor(void *obj, void *data) {
   csync_file_stat_t *st = NULL;
   CSYNC *ctx = NULL;
@@ -1595,6 +1644,13 @@ int csync_propagate_files(CSYNC *ctx) {
       break;
   }
 
+  /* If there is a overall progress callback set, count the number of files first. */
+  if (ctx->callbacks.overall_progress_cb) {
+      if (c_rbtree_walk(tree, (void *) ctx, _csync_propagation_file_count_visitor) < 0) {
+          return -1;
+      }
+  }
+
   if (c_rbtree_walk(tree, (void *) ctx, _csync_propagation_file_visitor) < 0) {
     return -1;
   }

src/csync_statedb.c

@@ -305,7 +305,7 @@ int csync_statedb_write(CSYNC *ctx) {
   }
 
   /* progress info */
-  if (csync_statedb_write_progressinfo(ctx, ctx->progress) < 0) {
+  if (csync_statedb_write_progressinfo(ctx, ctx->progress_info) < 0) {
     return -1;
   }