mirrors/nextcloud-desktop
1
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2024-11-26 15:06:08 +03:00
Code Issues Projects Releases Packages Wiki Activity

Improved database access code for csync.

Browse source 
- Use precompiled database statements
- Store them in the csync context struct
- Hand whole context as parameter to db access functions
This commit is contained in:
Klaas Freitag 2014-03-19 16:22:25 +01:00
parent f21dd05c2d
commit c3cee3426e
6 changed files with 162 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
csync/src/csync.c
View file

@ -563,7 +563,7 @@ int csync_commit(CSYNC *ctx) {
ctx->status_code = CSYNC_STATUS_OK;
if (ctx->statedb.db != NULL
&& csync_statedb_close(ctx->statedb.file, ctx->statedb.db, 0) < 0) {
&& csync_statedb_close(ctx, 0) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ERR: closing of statedb failed.");
rc = -1;
}
@ -618,7 +618,7 @@ int csync_destroy(CSYNC *ctx) {
ctx->status_code = CSYNC_STATUS_OK;
if (ctx->statedb.db != NULL
&& csync_statedb_close(ctx->statedb.file, ctx->statedb.db, 0) < 0) {
&& csync_statedb_close(ctx, 0) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ERR: closing of statedb failed.");
rc = -1;
}

4
csync/src/csync_private.h
View file

@ -106,6 +106,10 @@ struct csync_s {
sqlite3 *db;
int exists;
int disabled;
sqlite3_stmt* by_hash_stmt;
sqlite3_stmt* by_fileid_stmt;
sqlite3_stmt* by_inode_stmt;
} statedb;
struct {

4
csync/src/csync_reconcile.c
View file

@ -101,11 +101,11 @@ static int _csync_merge_algorithm_visitor(void *obj, void *data) {
case CSYNC_INSTRUCTION_EVAL_RENAME:
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx->statedb.db, cur->inode);
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s",
cur->inode, tmp ? "true":"false");
} else if( ctx->current == REMOTE_REPLICA ) {
tmp = csync_statedb_get_stat_by_file_id(ctx->statedb.db, cur->file_id);
tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s",
cur->file_id, tmp ? "true":"false");
} else {

256
csync/src/csync_statedb.c
View file

@ -48,9 +48,6 @@
#include "csync_rename.h"
#define BUF_SIZE 16
#define HASH_QUERY "SELECT * FROM metadata WHERE phash=?1"
static sqlite3_stmt* _by_hash_stmt = NULL;
void csync_set_statedb_exists(CSYNC *ctx, int val) {
ctx->statedb.exists = val;
@ -207,6 +204,10 @@ int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb) {
char *statedb_tmp = NULL;
sqlite3 *db = NULL;
if( !ctx ) {
return -1;
}
/* csync_statedb_check tries to open the statedb and creates it in case
* its not there.
*/
@ -277,21 +278,36 @@ out:
return rc;
}
int csync_statedb_close(const char *statedb, sqlite3 *db, int jwritten) {
int csync_statedb_close(CSYNC *ctx, int jwritten) {
char *statedb_tmp = NULL;
mbchar_t* wstatedb_tmp = NULL;
int rc = 0;
mbchar_t *mb_statedb = NULL;
if (!ctx) {
return -1;
}
/* deallocate query resources */
rc = sqlite3_finalize(_by_hash_stmt);
_by_hash_stmt = NULL;
if( ctx->statedb.by_hash_stmt ) {
rc = sqlite3_finalize(ctx->statedb.by_hash_stmt);
ctx->statedb.by_hash_stmt = NULL;
}
if( ctx->statedb.by_fileid_stmt ) {
rc = sqlite3_finalize(ctx->statedb.by_fileid_stmt);
ctx->statedb.by_fileid_stmt = NULL;
}
if( ctx->statedb.by_inode_stmt ) {
rc = sqlite3_finalize(ctx->statedb.by_inode_stmt);
ctx->statedb.by_inode_stmt = NULL;
}
/* close the temporary database */
sqlite3_close(db);
sqlite3_close(ctx->statedb.db);
if (asprintf(&statedb_tmp, "%s.ctmp", statedb) < 0) {
if (asprintf(&statedb_tmp, "%s.ctmp", ctx->statedb.file) < 0) {
return -1;
}
@ -309,10 +325,10 @@ int csync_statedb_close(const char *statedb, sqlite3 *db, int jwritten) {
*/
if (_csync_statedb_check(statedb_tmp) >= 0) {
/* New statedb is valid. */
mb_statedb = c_utf8_to_locale(statedb);
mb_statedb = c_utf8_to_locale(ctx->statedb.file);
/* Move the tmp-db to the real one. */
if (c_rename(statedb_tmp, statedb) < 0) {
if (c_rename(statedb_tmp, ctx->statedb.file) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"Renaming tmp db to original db failed. (errno=%d)", errno);
rc = -1;
@ -352,7 +368,10 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
int column_count;
int len;
if( ! stmt ) return -1;
if( ! stmt ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Fatal: Statement is NULL.");
return SQLITE_ERROR;
}
column_count = sqlite3_column_count(stmt);
@ -366,7 +385,7 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
len = sqlite3_column_int(stmt, 1);
*st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (*st == NULL) {
return -1;
return SQLITE_NOMEM;
}
/* clear the whole structure */
ZERO_STRUCTP(*st);
@ -399,177 +418,136 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
}
/* caller must free the memory */
csync_file_stat_t *csync_statedb_get_stat_by_hash(sqlite3 *db,
csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
uint64_t phash)
{
csync_file_stat_t *st = NULL;
int rc;
if( _by_hash_stmt == NULL ) {
rc = sqlite3_prepare_v2(db, HASH_QUERY, strlen(HASH_QUERY), &_by_hash_stmt, NULL);
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for hash query.");
if( !ctx ) {
return NULL;
}
}
if( _by_hash_stmt == NULL ) {
if( ctx->statedb.by_hash_stmt == NULL ) {
const char *hash_query = "SELECT * FROM metadata WHERE phash=?1";
rc = sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL);
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for hash query.");
return NULL;
}
}
if( ctx->statedb.by_hash_stmt == NULL ) {
return NULL;
}
sqlite3_bind_int64(_by_hash_stmt, 1, (long long signed int)phash);
sqlite3_bind_int64(ctx->statedb.by_hash_stmt, 1, (long long signed int)phash);
if( _csync_file_stat_from_metadata_table(&st, _by_hash_stmt) < 0 ) {
if( _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_hash_stmt) < 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata!");
}
sqlite3_reset(_by_hash_stmt);
sqlite3_reset(ctx->statedb.by_hash_stmt);
return st;
}
csync_file_stat_t *csync_statedb_get_stat_by_file_id( sqlite3 *db,
const char *file_id ) {
csync_file_stat_t *st = NULL;
c_strlist_t *result = NULL;
char *stmt = NULL;
size_t len = 0;
csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
const char *file_id ) {
csync_file_stat_t *st = NULL;
int rc = 0;
if (!file_id) {
return 0;
}
if (c_streq(file_id, "")) {
return 0;
}
stmt = sqlite3_mprintf("SELECT * FROM metadata WHERE fileid='%q'",
file_id);
if (!file_id) {
return 0;
}
if (c_streq(file_id, "")) {
return 0;
}
if (stmt == NULL) {
return NULL;
}
if( !ctx ) {
return NULL;
}
result = csync_statedb_query(db, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
}
if( ctx->statedb.by_fileid_stmt == NULL ) {
const char *query = "SELECT * FROM metadata WHERE fileid='?1'";
if (result->count <= 6) {
c_strlist_destroy(result);
return NULL;
}
rc = sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL);
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for file id query.");
return NULL;
}
}
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
len = strlen(result->vector[2]);
st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (st == NULL) {
c_strlist_destroy(result);
return NULL;
}
/* clear the whole structure */
ZERO_STRUCTP(st);
/* bind the query value */
sqlite3_bind_text(ctx->statedb.by_fileid_stmt, 1, file_id, -1, SQLITE_STATIC);
st->phash = atoll(result->vector[0]);
st->pathlen = atoi(result->vector[1]);
memcpy(st->path, (len ? result->vector[2] : ""), len + 1);
st->inode = atoll(result->vector[3]);
st->uid = atoi(result->vector[4]);
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
st->type = atoi(result->vector[8]);
if( result->vector[9] )
st->etag = c_strdup(result->vector[9]);
csync_vio_set_file_id(st->file_id, file_id);
c_strlist_destroy(result);
return st;
}
if( _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_fileid_stmt) < 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata!");
}
// clear the resources used by the statement.
sqlite3_reset(ctx->statedb.by_fileid_stmt);
return st;
}
/* caller must free the memory */
csync_file_stat_t *csync_statedb_get_stat_by_inode(sqlite3 *db,
uint64_t inode) {
csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
uint64_t inode)
{
csync_file_stat_t *st = NULL;
c_strlist_t *result = NULL;
char *stmt = NULL;
size_t len = 0;
int rc;
if (!inode) {
return NULL;
}
stmt = sqlite3_mprintf("SELECT * FROM metadata WHERE inode='%lld'",
(long long signed int) inode);
if (stmt == NULL) {
if( !ctx ) {
return NULL;
}
if( ctx->statedb.by_inode_stmt == NULL ) {
const char *inode_query = "SELECT * FROM metadata WHERE inode=?1";
rc = sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL);
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for inode query.");
return NULL;
}
}
if( ctx->statedb.by_inode_stmt == NULL ) {
return NULL;
}
result = csync_statedb_query(db, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
sqlite3_bind_int64(ctx->statedb.by_inode_stmt, 1, (long long signed int)inode);
if( _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_inode_stmt) < 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata by inode!");
}
if (result->count <= 6) {
c_strlist_destroy(result);
return NULL;
}
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
len = strlen(result->vector[2]);
st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (st == NULL) {
c_strlist_destroy(result);
return NULL;
}
/* clear the whole structure */
ZERO_STRUCTP(st);
st->phash = atoll(result->vector[0]);
st->pathlen = atoi(result->vector[1]);
memcpy(st->path, (len ? result->vector[2] : ""), len + 1);
st->inode = atoll(result->vector[3]);
st->uid = atoi(result->vector[4]);
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
st->type = atoi(result->vector[8]);
if( result->vector[9] )
st->etag = c_strdup(result->vector[9]);
csync_vio_set_file_id( st->file_id, result->vector[10]);
c_strlist_destroy(result);
sqlite3_reset(ctx->statedb.by_inode_stmt);
return st;
}
/* Get the etag. (it is called unique id for legacy reason
* and it is the field md5 in the database for legacy reason */
char *csync_statedb_get_uniqId( CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf ) {
/* Get the etag. */
char *csync_statedb_get_etag( CSYNC *ctx, uint64_t jHash ) {
char *ret = NULL;
c_strlist_t *result = NULL;
char *stmt = NULL;
(void)buf;
csync_file_stat_t *fs = NULL;
if( !ctx ) {
return NULL;
}
if( ! csync_get_statedb_exists(ctx)) return ret;
stmt = sqlite3_mprintf("SELECT md5, fileid FROM metadata WHERE phash='%lld'", jHash);
result = csync_statedb_query(ctx->statedb.db, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
fs = csync_statedb_get_stat_by_hash(ctx, jHash );
if( fs ) {
if( fs->etag ) {
ret = c_strdup(fs->etag);
}
csync_file_stat_free(fs);
}
if (result->count == 2) {
ret = c_strdup( result->vector[0] );
csync_vio_file_stat_set_file_id(buf, result->vector[1]);
}
c_strlist_destroy(result);
return ret;
}
@ -586,6 +564,10 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
return -1;
}
if( !ctx ) {
return -1;
}
rc = sqlite3_prepare_v2(ctx->statedb.db, BELOW_PATH_QUERY, -1, &stmt, NULL);
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for hash query.");

11
csync/src/csync_statedb.h
View file

@ -54,16 +54,15 @@ int csync_get_statedb_exists(CSYNC *ctx);
*/
int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb);
int csync_statedb_close(const char *statedb, sqlite3 *db, int jwritten);
int csync_statedb_close(CSYNC *ctx, int jwritten);
csync_file_stat_t *csync_statedb_get_stat_by_hash(sqlite3 *db, uint64_t phash);
csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash);
csync_file_stat_t *csync_statedb_get_stat_by_inode(sqlite3 *db, uint64_t inode);
csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode);
csync_file_stat_t *csync_statedb_get_stat_by_file_id( sqlite3 *db,
const char *file_id );
csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx, const char *file_id);
char *csync_statedb_get_uniqId(CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf);
char *csync_statedb_get_etag(CSYNC *ctx, uint64_t jHash);
/**
* @brief Query all files metadata inside and below a path.

50
csync/src/csync_update.c
View file

@ -168,7 +168,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
}
if (fs->mtime == 0) {
tmp = csync_statedb_get_stat_by_hash(ctx->statedb.db, h);
tmp = csync_statedb_get_stat_by_hash(ctx, h);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path);
if (tmp == NULL) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - not found in db, IGNORE!", path);
@ -204,7 +204,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
* does not change on rename.
*/
if (csync_get_statedb_exists(ctx)) {
tmp = csync_statedb_get_stat_by_hash(ctx->statedb.db, h);
tmp = csync_statedb_get_stat_by_hash(ctx, h);
if(tmp && tmp->phash == h ) { /* there is an entry in the database */
/* we have an update! */
@ -238,6 +238,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
&& c_streq(fs->file_id, tmp->file_id)) {
/* If both etag and file id are equal for a directory, read all contents from
* the database. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Reading from database: %s", path);
ctx->remote.read_from_db = true;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
@ -246,7 +247,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
/* check if it's a file and has been renamed */
if (ctx->current == LOCAL_REPLICA) {
tmp = csync_statedb_get_stat_by_inode(ctx->statedb.db, fs->inode);
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
/* translate the file type between the two stat types csync has. */
if( tmp && tmp->type == 0 ) {
@ -273,7 +274,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
}
} else {
/* Remote Replica Rename check */
tmp = csync_statedb_get_stat_by_file_id(ctx->statedb.db, fs->file_id);
tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id);
if(tmp ) { /* tmp existing at all */
if ((tmp->type == CSYNC_FTW_TYPE_DIR && fs->type != CSYNC_VIO_FILE_TYPE_DIRECTORY) ||
(tmp->type == CSYNC_FTW_TYPE_FILE && fs->type != CSYNC_VIO_FILE_TYPE_REGULAR)) {
@ -359,9 +360,9 @@ fastout: /* target if the file information is read from database into st */
default:
break;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "==> %s %s - hash %llu, mtime: %llu, fileId: %s",
csync_instruction_str(st->instruction), path, (unsigned long long ) h,
(unsigned long long) fs->mtime, fs->file_id);
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: %s <<=", st->path,
csync_instruction_str(st->instruction));
return 0;
}
@ -373,18 +374,18 @@ int csync_walker(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs,
uint64_t h;
if (ctx->abort) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_INFO, "Aborted!");
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Aborted!");
ctx->status_code = CSYNC_STATUS_ABORTED;
return -1;
}
switch (flag) {
case CSYNC_FTW_FLAG_FILE:
// CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s", file);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s", file);
type = CSYNC_FTW_TYPE_FILE;
break;
case CSYNC_FTW_FLAG_DIR: /* enter directory */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "directory: %s", file);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "directory: %s", file);
type = CSYNC_FTW_TYPE_DIR;
break;
case CSYNC_FTW_FLAG_NSTAT: /* not statable file */
@ -394,7 +395,7 @@ int csync_walker(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs,
if( h == 0 ) {
return 0;
}
st = csync_statedb_get_stat_by_hash(ctx->statedb.db, h);
st = csync_statedb_get_stat_by_hash(ctx, h);
if( !st ) {
return 0;
}
@ -420,20 +421,23 @@ int csync_walker(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs,
return rc;
}
static void fill_tree_from_db(CSYNC *ctx, const char *uri)
static bool fill_tree_from_db(CSYNC *ctx, const char *uri)
{
const char *path = NULL;
if( strlen(uri) < strlen(ctx->remote.uri)+1) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "name does not contain remote uri!");
return;
return false;
}
path = uri + strlen(ctx->remote.uri)+1;
if( csync_statedb_get_below_path(ctx, path) < 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "StateDB could not be read!");
return false;
}
return true;
}
/* File tree walker */
@ -463,7 +467,11 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
// if the etag of this dir is still the same, its content is restored from the
// database.
if( do_read_from_db ) {
fill_tree_from_db(ctx, uri);
if( ! fill_tree_from_db(ctx, uri) ) {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR;
goto error;
}
goto done;
}
@ -579,16 +587,18 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
char *etag = NULL;
int len = strlen( path );
uint64_t h = c_jhash64((uint8_t *) path, len, 0);
etag = csync_statedb_get_uniqId( ctx, h, fs );
etag = csync_statedb_get_etag( ctx, h );
if( etag ) {
SAFE_FREE(fs->etag);
fs->etag = etag;
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_ETAG;
}
if( c_streq(etag, "")) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database is EMPTY: %s", path);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database: %s -> %s", path, fs->etag ? fs->etag : "<NULL>" );
if( c_streq(etag, "")) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database is EMPTY: %s", path);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database: %s -> %s", path, fs->etag ? fs->etag : "<NULL>" );
}
}
}