mirror of
https://github.com/nextcloud/desktop.git
synced 2024-10-27 23:17:13 +03:00
Remove the usage of phash in csync
Only store the path since they represent the same thing, and do the phash conversion during DB lookup like done in libsync. We could get rid of everything since we also have an index on the path column, but since it's the primary key this makes the migration non-trivial.
This commit is contained in:
parent
2276df1eec
commit
3bc1f63b0a
6 changed files with 24 additions and 34 deletions
|
@ -160,7 +160,6 @@ enum csync_ftw_type_e {
|
||||||
typedef struct csync_file_stat_s csync_file_stat_t;
|
typedef struct csync_file_stat_s csync_file_stat_t;
|
||||||
|
|
||||||
struct OCSYNC_EXPORT csync_file_stat_s {
|
struct OCSYNC_EXPORT csync_file_stat_s {
|
||||||
uint64_t phash;
|
|
||||||
time_t modtime;
|
time_t modtime;
|
||||||
int64_t size;
|
int64_t size;
|
||||||
uint64_t inode;
|
uint64_t inode;
|
||||||
|
@ -189,8 +188,7 @@ struct OCSYNC_EXPORT csync_file_stat_s {
|
||||||
enum csync_instructions_e instruction; /* u32 */
|
enum csync_instructions_e instruction; /* u32 */
|
||||||
|
|
||||||
csync_file_stat_s()
|
csync_file_stat_s()
|
||||||
: phash(0)
|
: modtime(0)
|
||||||
, modtime(0)
|
|
||||||
, size(0)
|
, size(0)
|
||||||
, inode(0)
|
, inode(0)
|
||||||
, type(CSYNC_FTW_TYPE_SKIP)
|
, type(CSYNC_FTW_TYPE_SKIP)
|
||||||
|
|
|
@ -231,7 +231,7 @@ int csync_statedb_close(CSYNC *ctx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
#define METADATA_QUERY \
|
#define METADATA_QUERY \
|
||||||
"phash, path, inode, modtime, type, md5, fileid, remotePerm, " \
|
"path, inode, modtime, type, md5, fileid, remotePerm, " \
|
||||||
"filesize, ignoredChildrenRemote, " \
|
"filesize, ignoredChildrenRemote, " \
|
||||||
"contentchecksumtype.name || ':' || contentChecksum " \
|
"contentchecksumtype.name || ':' || contentChecksum " \
|
||||||
"FROM metadata " \
|
"FROM metadata " \
|
||||||
|
@ -251,25 +251,23 @@ static int _csync_file_stat_from_metadata_table( std::unique_ptr<csync_file_stat
|
||||||
}
|
}
|
||||||
|
|
||||||
// Callers should all use METADATA_QUERY for their column list.
|
// Callers should all use METADATA_QUERY for their column list.
|
||||||
assert(sqlite3_column_count(stmt) == 11);
|
assert(sqlite3_column_count(stmt) == 10);
|
||||||
|
|
||||||
SQLITE_BUSY_HANDLED( sqlite3_step(stmt) );
|
SQLITE_BUSY_HANDLED( sqlite3_step(stmt) );
|
||||||
|
|
||||||
if( rc == SQLITE_ROW ) {
|
if( rc == SQLITE_ROW ) {
|
||||||
st.reset(new csync_file_stat_t);
|
st.reset(new csync_file_stat_t);
|
||||||
|
|
||||||
/* The query suceeded so use the phash we pass to the function. */
|
st->path = (char*)sqlite3_column_text(stmt, 0);
|
||||||
st->phash = sqlite3_column_int64(stmt, 0);
|
st->inode = sqlite3_column_int64(stmt, 1);
|
||||||
st->path = (char*)sqlite3_column_text(stmt, 1);
|
st->modtime = strtoul((char*)sqlite3_column_text(stmt, 2), NULL, 10);
|
||||||
st->inode = sqlite3_column_int64(stmt, 2);
|
st->type = static_cast<enum csync_ftw_type_e>(sqlite3_column_int(stmt, 3));
|
||||||
st->modtime = strtoul((char*)sqlite3_column_text(stmt, 3), NULL, 10);
|
st->etag = (char*)sqlite3_column_text(stmt, 4);
|
||||||
st->type = static_cast<enum csync_ftw_type_e>(sqlite3_column_int(stmt, 4));
|
st->file_id = (char*)sqlite3_column_text(stmt, 5);
|
||||||
st->etag = (char*)sqlite3_column_text(stmt, 5);
|
st->remotePerm = (char*)sqlite3_column_text(stmt, 6);
|
||||||
st->file_id = (char*)sqlite3_column_text(stmt, 6);
|
st->size = sqlite3_column_int64(stmt, 7);
|
||||||
st->remotePerm = (char*)sqlite3_column_text(stmt, 7);
|
st->has_ignored_files = sqlite3_column_int(stmt, 8);
|
||||||
st->size = sqlite3_column_int64(stmt, 8);
|
st->checksumHeader = (char *)sqlite3_column_text(stmt, 9);
|
||||||
st->has_ignored_files = sqlite3_column_int(stmt, 9);
|
|
||||||
st->checksumHeader = (char *)sqlite3_column_text(stmt, 10);
|
|
||||||
} else {
|
} else {
|
||||||
if( rc != SQLITE_DONE ) {
|
if( rc != SQLITE_DONE ) {
|
||||||
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Query results in %d", rc);
|
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Query results in %d", rc);
|
||||||
|
@ -279,8 +277,7 @@ static int _csync_file_stat_from_metadata_table( std::unique_ptr<csync_file_stat
|
||||||
}
|
}
|
||||||
|
|
||||||
/* caller must free the memory */
|
/* caller must free the memory */
|
||||||
std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_hash(CSYNC *ctx,
|
std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_path(CSYNC *ctx, const QByteArray &path)
|
||||||
uint64_t phash)
|
|
||||||
{
|
{
|
||||||
std::unique_ptr<csync_file_stat_t> st;
|
std::unique_ptr<csync_file_stat_t> st;
|
||||||
int rc;
|
int rc;
|
||||||
|
@ -304,6 +301,7 @@ std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_hash(CSYNC *ctx,
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint64_t phash = c_jhash64((const uint8_t*)path.constData(), path.size(), 0);
|
||||||
sqlite3_bind_int64(ctx->statedb.by_hash_stmt, 1, (long long signed int)phash);
|
sqlite3_bind_int64(ctx->statedb.by_hash_stmt, 1, (long long signed int)phash);
|
||||||
|
|
||||||
rc = _csync_file_stat_from_metadata_table(st, ctx->statedb.by_hash_stmt);
|
rc = _csync_file_stat_from_metadata_table(st, ctx->statedb.by_hash_stmt);
|
||||||
|
|
|
@ -56,7 +56,7 @@ OCSYNC_EXPORT int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **
|
||||||
|
|
||||||
OCSYNC_EXPORT int csync_statedb_close(CSYNC *ctx);
|
OCSYNC_EXPORT int csync_statedb_close(CSYNC *ctx);
|
||||||
|
|
||||||
OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash);
|
OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_path(CSYNC *ctx, const QByteArray &path);
|
||||||
|
|
||||||
OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode);
|
OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode);
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,6 @@
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
|
|
||||||
#include "c_lib.h"
|
#include "c_lib.h"
|
||||||
#include "c_jhash.h"
|
|
||||||
|
|
||||||
#include "csync_private.h"
|
#include "csync_private.h"
|
||||||
#include "csync_exclude.h"
|
#include "csync_exclude.h"
|
||||||
|
@ -172,14 +171,14 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f
|
||||||
* does not change on rename.
|
* does not change on rename.
|
||||||
*/
|
*/
|
||||||
if (csync_get_statedb_exists(ctx)) {
|
if (csync_get_statedb_exists(ctx)) {
|
||||||
tmp = csync_statedb_get_stat_by_hash(ctx, fs->phash);
|
tmp = csync_statedb_get_stat_by_path(ctx, fs->path);
|
||||||
|
|
||||||
if(_last_db_return_error(ctx)) {
|
if(_last_db_return_error(ctx)) {
|
||||||
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
|
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(tmp && tmp->phash == fs->phash ) { /* there is an entry in the database */
|
if(tmp && tmp->path == fs->path ) { /* there is an entry in the database */
|
||||||
/* we have an update! */
|
/* we have an update! */
|
||||||
CSYNC_LOG(CSYNC_LOG_PRIORITY_INFO, "Database entry found, compare: %" PRId64 " <-> %" PRId64
|
CSYNC_LOG(CSYNC_LOG_PRIORITY_INFO, "Database entry found, compare: %" PRId64 " <-> %" PRId64
|
||||||
", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64
|
", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64
|
||||||
|
@ -594,8 +593,6 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
|
||||||
// "len + 1" to include the slash in-between.
|
// "len + 1" to include the slash in-between.
|
||||||
dirent->path = dirent->path.mid(strlen(ctx->local.uri) + 1);
|
dirent->path = dirent->path.mid(strlen(ctx->local.uri) + 1);
|
||||||
}
|
}
|
||||||
// We calculate the phash using the relative path.
|
|
||||||
dirent->phash = c_jhash64((const uint8_t*)dirent->path.constData(), dirent->path.size(), 0);
|
|
||||||
|
|
||||||
previous_fs = ctx->current_fs;
|
previous_fs = ctx->current_fs;
|
||||||
bool recurse = dirent->type == CSYNC_FTW_TYPE_DIR;
|
bool recurse = dirent->type == CSYNC_FTW_TYPE_DIR;
|
||||||
|
|
|
@ -155,7 +155,6 @@ static void check_csync_statedb_insert_metadata(void **state)
|
||||||
for (i = 0; i < 100; i++) {
|
for (i = 0; i < 100; i++) {
|
||||||
st.reset(new csync_file_stat_t);
|
st.reset(new csync_file_stat_t);
|
||||||
st->path = QString("file_%1").arg(i).toUtf8();
|
st->path = QString("file_%1").arg(i).toUtf8();
|
||||||
st->phash = i;
|
|
||||||
|
|
||||||
csync->local.files[st->path] = std::move(st);
|
csync->local.files[st->path] = std::move(st);
|
||||||
}
|
}
|
||||||
|
@ -173,7 +172,6 @@ static void check_csync_statedb_write(void **state)
|
||||||
for (i = 0; i < 100; i++) {
|
for (i = 0; i < 100; i++) {
|
||||||
st.reset(new csync_file_stat_t);
|
st.reset(new csync_file_stat_t);
|
||||||
st->path = QString("file_%1").arg(i).toUtf8();
|
st->path = QString("file_%1").arg(i).toUtf8();
|
||||||
st->phash = i;
|
|
||||||
|
|
||||||
csync->local.files[st->path] = std::move(st);
|
csync->local.files[st->path] = std::move(st);
|
||||||
assert_int_equal(rc, 0);
|
assert_int_equal(rc, 0);
|
||||||
|
@ -184,12 +182,12 @@ static void check_csync_statedb_write(void **state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void check_csync_statedb_get_stat_by_hash_not_found(void **state)
|
static void check_csync_statedb_get_stat_by_path_not_found(void **state)
|
||||||
{
|
{
|
||||||
CSYNC *csync = (CSYNC*)*state;
|
CSYNC *csync = (CSYNC*)*state;
|
||||||
std::unique_ptr<csync_file_stat_t> tmp;
|
std::unique_ptr<csync_file_stat_t> tmp;
|
||||||
|
|
||||||
tmp = csync_statedb_get_stat_by_hash(csync, (uint64_t) 666);
|
tmp = csync_statedb_get_stat_by_path(csync, "666");
|
||||||
assert_null(tmp.get());
|
assert_null(tmp.get());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -210,7 +208,7 @@ int torture_run_tests(void)
|
||||||
cmocka_unit_test_setup_teardown(check_csync_statedb_drop_tables, setup, teardown),
|
cmocka_unit_test_setup_teardown(check_csync_statedb_drop_tables, setup, teardown),
|
||||||
cmocka_unit_test_setup_teardown(check_csync_statedb_insert_metadata, setup, teardown),
|
cmocka_unit_test_setup_teardown(check_csync_statedb_insert_metadata, setup, teardown),
|
||||||
cmocka_unit_test_setup_teardown(check_csync_statedb_write, setup, teardown),
|
cmocka_unit_test_setup_teardown(check_csync_statedb_write, setup, teardown),
|
||||||
cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_hash_not_found, setup_db, teardown),
|
cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_path_not_found, setup_db, teardown),
|
||||||
cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_inode_not_found, setup_db, teardown),
|
cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_inode_not_found, setup_db, teardown),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -27,9 +27,8 @@ private slots:
|
||||||
}
|
}
|
||||||
|
|
||||||
void testFullResult() {
|
void testFullResult() {
|
||||||
std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_hash( _ctx, 2081025720555645157 );
|
std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_path( _ctx, "test2/zu/zuzu" );
|
||||||
QVERIFY(st.get());
|
QVERIFY(st.get());
|
||||||
QCOMPARE( QString::number(st->phash), QString::number(2081025720555645157) );
|
|
||||||
QCOMPARE( QString::fromUtf8(st->path), QLatin1String("test2/zu/zuzu") );
|
QCOMPARE( QString::fromUtf8(st->path), QLatin1String("test2/zu/zuzu") );
|
||||||
QCOMPARE( QString::number(st->inode), QString::number(1709554));
|
QCOMPARE( QString::number(st->inode), QString::number(1709554));
|
||||||
QCOMPARE( QString::number(st->modtime), QString::number(1384415006));
|
QCOMPARE( QString::number(st->modtime), QString::number(1384415006));
|
||||||
|
@ -40,11 +39,11 @@ private slots:
|
||||||
}
|
}
|
||||||
|
|
||||||
void testByHash() {
|
void testByHash() {
|
||||||
std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_hash(_ctx, -7147279406142960289);
|
std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_path(_ctx, "documents/c1");
|
||||||
QVERIFY(st.get());
|
QVERIFY(st.get());
|
||||||
QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1"));
|
QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1"));
|
||||||
|
|
||||||
st = csync_statedb_get_stat_by_hash(_ctx, 5426481156826978940);
|
st = csync_statedb_get_stat_by_path(_ctx, "documents/c1/c2");
|
||||||
QVERIFY(st.get());
|
QVERIFY(st.get());
|
||||||
QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1/c2"));
|
QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1/c2"));
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue